/******************************************************************** * COPYRIGHT: * Copyright (c) 2001-2004, International Business Machines Corporation and * others. All Rights Reserved. ********************************************************************/ /******************************************************************************* * * File cmsccoll.C * *******************************************************************************/ /** * These are the tests specific to ICU 1.8 and above, that I didn't know where * to fit. */ #include #include "unicode/utypes.h" #if !UCONFIG_NO_COLLATION #include "unicode/ucol.h" #include "unicode/ucoleitr.h" #include "unicode/uloc.h" #include "cintltst.h" #include "ccolltst.h" #include "callcoll.h" #include "unicode/ustring.h" #include "string.h" #include "ucol_imp.h" #include "ucol_tok.h" #include "cmemory.h" #include "cstring.h" #include "uassert.h" #include "unicode/parseerr.h" #include "unicode/ucnv.h" #include "uparse.h" #define LEN(a) (sizeof(a)/sizeof(a[0])) #define MAX_TOKEN_LEN 16 typedef int tst_strcoll(void *collator, const int object, const UChar *source, const int sLen, const UChar *target, const int tLen); const static char cnt1[][10] = { "AA", "AC", "AZ", "AQ", "AB", "ABZ", "ABQ", "Z", "ABC", "Q", "B" }; const static char cnt2[][10] = { "DA", "DAD", "DAZ", "MAR", "Z", "DAVIS", "MARK", "DAV", "DAVI" }; static void IncompleteCntTest(void) { UErrorCode status = U_ZERO_ERROR; UChar temp[90]; UChar t1[90]; UChar t2[90]; UCollator *coll = NULL; uint32_t i = 0, j = 0; uint32_t size = 0; u_uastrcpy(temp, " & Z < ABC < Q < B"); coll = ucol_openRules(temp, u_strlen(temp), UCOL_OFF, UCOL_DEFAULT_STRENGTH, NULL,&status); if(U_SUCCESS(status)) { size = sizeof(cnt1)/sizeof(cnt1[0]); for(i = 0; i < size-1; i++) { for(j = i+1; j < size; j++) { UCollationElements *iter; u_uastrcpy(t1, cnt1[i]); u_uastrcpy(t2, cnt1[j]); doTest(coll, t1, t2, UCOL_LESS); /* synwee : added collation element iterator test */ iter = ucol_openElements(coll, t2, u_strlen(t2), &status); if (U_FAILURE(status)) { log_err("Creation of iterator failed\n"); break; } backAndForth(iter); ucol_closeElements(iter); } } } ucol_close(coll); u_uastrcpy(temp, " & Z < DAVIS < MARK 0x0400 && *p<0x0500)?0x00e1:0x491; UChar preQ = (*p>0x0400 && *p<0x0500)?0x0041:0x413; */ /*log_verbose("Testing primary\n");*/ doTest(col, p, q, UCOL_LESS); /* UCollationResult result = ucol_strcoll(col,p,u_strlen(p),q,u_strlen(q)); if(result!=UCOL_LESS){ aescstrdup(p,utfSource,256); aescstrdup(q,utfTarget,256); fprintf(file,"Primary failed source: %s target: %s \n", utfSource,utfTarget); } */ source[0] = preP; u_strcpy(source+1,p); target[0] = preQ; u_strcpy(target+1,q); doTest(col, source, target, UCOL_LESS); /* fprintf(file,"Primary swamps 2nd failed source: %s target: %s \n", utfSource,utfTarget); */ } static void testSecondary(UCollator* col, const UChar* p,const UChar* q){ UChar source[256] = { '\0'}; UChar target[256] = { '\0'}; /*log_verbose("Testing secondary\n");*/ doTest(col, p, q, UCOL_LESS); /* fprintf(file,"secondary failed source: %s target: %s \n", utfSource,utfTarget); */ source[0] = 0x0053; u_strcpy(source+1,p); target[0]= 0x0073; u_strcpy(target+1,q); doTest(col, source, target, UCOL_LESS); /* fprintf(file,"secondary swamps 3rd failed source: %s target: %s \n",utfSource,utfTarget); */ u_strcpy(source,p); source[u_strlen(p)] = 0x62; source[u_strlen(p)+1] = 0; u_strcpy(target,q); target[u_strlen(q)] = 0x61; target[u_strlen(q)+1] = 0; doTest(col, source, target, UCOL_GREATER); /* fprintf(file,"secondary is swamped by 1 failed source: %s target: %s \n",utfSource,utfTarget); */ } static void testTertiary(UCollator* col, const UChar* p,const UChar* q){ UChar source[256] = { '\0'}; UChar target[256] = { '\0'}; /*log_verbose("Testing tertiary\n");*/ doTest(col, p, q, UCOL_LESS); /* fprintf(file,"Tertiary failed source: %s target: %s \n",utfSource,utfTarget); */ source[0] = 0x0020; u_strcpy(source+1,p); target[0]= 0x002D; u_strcpy(target+1,q); doTest(col, source, target, UCOL_LESS); /* fprintf(file,"Tertiary swamps 4th failed source: %s target: %s \n", utfSource,utfTarget); */ u_strcpy(source,p); source[u_strlen(p)] = 0xE0; source[u_strlen(p)+1] = 0; u_strcpy(target,q); target[u_strlen(q)] = 0x61; target[u_strlen(q)+1] = 0; doTest(col, source, target, UCOL_GREATER); /* fprintf(file,"Tertiary is swamped by 3rd failed source: %s target: %s \n",utfSource,utfTarget); */ } static void testEquality(UCollator* col, const UChar* p,const UChar* q){ /* UChar source[256] = { '\0'}; UChar target[256] = { '\0'}; */ doTest(col, p, q, UCOL_EQUAL); /* fprintf(file,"Primary failed source: %s target: %s \n", utfSource,utfTarget); */ } static void testCollator(UCollator *coll, UErrorCode *status) { const UChar *rules = NULL, *current = NULL; int32_t ruleLen = 0; uint32_t strength = 0; uint32_t chOffset = 0; uint32_t chLen = 0; uint32_t exOffset = 0; uint32_t exLen = 0; uint32_t prefixOffset = 0; uint32_t prefixLen = 0; uint32_t firstEx = 0; /* uint32_t rExpsLen = 0; */ uint32_t firstLen = 0; UBool varT = FALSE; UBool top_ = TRUE; uint16_t specs = 0; UBool startOfRules = TRUE; UBool lastReset = FALSE; UBool before = FALSE; uint32_t beforeStrength = 0; UColTokenParser src; UColOptionSet opts; UChar first[256]; UChar second[256]; UChar tempB[256]; uint32_t tempLen; UChar *rulesCopy = NULL; UParseError parseError; src.opts = &opts; rules = ucol_getRules(coll, &ruleLen); if(U_SUCCESS(*status) && ruleLen > 0) { rulesCopy = (UChar *)malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar)); uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar)); src.current = src.source = rulesCopy; src.end = rulesCopy+ruleLen; src.extraCurrent = src.end; src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE; *first = *second = 0; while ((current = ucol_tok_parseNextToken(&src, startOfRules,&parseError, status)) != NULL) { strength = src.parsedToken.strength; chOffset = src.parsedToken.charsOffset; chLen = src.parsedToken.charsLen; exOffset = src.parsedToken.extensionOffset; exLen = src.parsedToken.extensionLen; prefixOffset = src.parsedToken.prefixOffset; prefixLen = src.parsedToken.prefixLen; specs = src.parsedToken.flags; startOfRules = FALSE; varT = (UBool)((specs & UCOL_TOK_VARIABLE_TOP) != 0); top_ = (UBool)((specs & UCOL_TOK_TOP) != 0); if(top_) { /* if reset is on top, the sequence is broken. We should have an empty string */ second[0] = 0; } else { u_strncpy(second,rulesCopy+chOffset, chLen); second[chLen] = 0; if(exLen > 0 && firstEx == 0) { u_strncat(first, rulesCopy+exOffset, exLen); first[firstLen+exLen] = 0; } if(lastReset == TRUE && prefixLen != 0) { u_strncpy(first+prefixLen, first, firstLen); u_strncpy(first, rulesCopy+prefixOffset, prefixLen); first[firstLen+prefixLen] = 0; firstLen = firstLen+prefixLen; } if(before == TRUE) { /* swap first and second */ u_strcpy(tempB, first); u_strcpy(first, second); u_strcpy(second, tempB); tempLen = firstLen; firstLen = chLen; chLen = tempLen; tempLen = firstEx; firstEx = exLen; exLen = tempLen; if(beforeStrength < strength) { strength = beforeStrength; } } } lastReset = FALSE; switch(strength){ case UCOL_IDENTICAL: testEquality(coll,first,second); break; case UCOL_PRIMARY: testPrimary(coll,first,second); break; case UCOL_SECONDARY: testSecondary(coll,first,second); break; case UCOL_TERTIARY: testTertiary(coll,first,second); break; case UCOL_TOK_RESET: lastReset = TRUE; before = (UBool)((specs & UCOL_TOK_BEFORE) != 0); if(before) { beforeStrength = (specs & UCOL_TOK_BEFORE)-1; } break; default: break; } if(before == TRUE && strength != UCOL_TOK_RESET) { /* first and second were swapped */ before = FALSE; } else { firstLen = chLen; firstEx = exLen; u_strcpy(first, second); } } free(rulesCopy); } } static int ucaTest(void *collator, const int object, const UChar *source, const int sLen, const UChar *target, const int tLen) { UCollator *UCA = (UCollator *)collator; return ucol_strcoll(UCA, source, sLen, target, tLen); } /* static int winTest(void *collator, const int object, const UChar *source, const int sLen, const UChar *target, const int tLen) { #ifdef WIN32 LCID lcid = (LCID)collator; return CompareString(lcid, 0, source, sLen, target, tLen); #else return 0; #endif } */ static UCollationResult swampEarlier(tst_strcoll* func, void *collator, int opts, UChar s1, UChar s2, const UChar *s, const uint32_t sLen, const UChar *t, const uint32_t tLen) { UChar source[256] = {0}; UChar target[256] = {0}; source[0] = s1; u_strcpy(source+1, s); target[0] = s2; u_strcpy(target+1, t); return func(collator, opts, source, sLen+1, target, tLen+1); } static UCollationResult swampLater(tst_strcoll* func, void *collator, int opts, UChar s1, UChar s2, const UChar *s, const uint32_t sLen, const UChar *t, const uint32_t tLen) { UChar source[256] = {0}; UChar target[256] = {0}; u_strcpy(source, s); source[sLen] = s1; u_strcpy(target, t); target[tLen] = s2; return func(collator, opts, source, sLen+1, target, tLen+1); } static uint32_t probeStrength(tst_strcoll* func, void *collator, int opts, const UChar *s, const uint32_t sLen, const UChar *t, const uint32_t tLen, UCollationResult result) { /*UChar fPrimary = 0x6d;*/ /*UChar sPrimary = 0x6e;*/ UChar fSecondary = 0x310d; UChar sSecondary = 0x31a3; UChar fTertiary = 0x310f; UChar sTertiary = 0x31b7; UCollationResult oposite; if(result == UCOL_EQUAL) { return UCOL_IDENTICAL; } else if(result == UCOL_GREATER) { oposite = UCOL_LESS; } else { oposite = UCOL_GREATER; } if(swampEarlier(func, collator, opts, sSecondary, fSecondary, s, sLen, t, tLen) == result) { return UCOL_PRIMARY; } else if((swampEarlier(func, collator, opts, sTertiary, 0x310f, s, sLen, t, tLen) == result) && (swampEarlier(func, collator, opts, 0x310f, sTertiary, s, sLen, t, tLen) == result)) { return UCOL_SECONDARY; } else if((swampLater(func, collator, opts, sTertiary, fTertiary, s, sLen, t, tLen) == result) && (swampLater(func, collator, opts, fTertiary, sTertiary, s, sLen, t, tLen) == result)) { return UCOL_TERTIARY; } else if((swampLater(func, collator, opts, sTertiary, 0x310f, s, sLen, t, tLen) == oposite) && (swampLater(func, collator, opts, fTertiary, sTertiary, s, sLen, t, tLen) == oposite)) { return UCOL_QUATERNARY; } else { return UCOL_IDENTICAL; } } static char *getRelationSymbol(UCollationResult res, uint32_t strength, char *buffer) { uint32_t i = 0; if(res == UCOL_EQUAL || strength == 0xdeadbeef) { buffer[0] = '='; buffer[1] = '='; buffer[2] = '\0'; } else if(res == UCOL_GREATER) { for(i = 0; i maxOutputLength) { maxOutputLength = outputLength; U_ASSERT(outputLength < sizeof(output)); } log_verbose("%s\n", output); } /* static void printOutRules(const UChar *rules) { uint32_t len = u_strlen(rules); uint32_t i = 0; char toPrint; uint32_t line = 0; fprintf(stdout, "Rules:"); for(i = 0; i=0x20) { toPrint = (char)rules[i]; if(toPrint == '&') { line = 1; fprintf(stdout, "\n&"); } else if(toPrint == ';') { fprintf(stdout, "<<"); line+=2; } else if(toPrint == ',') { fprintf(stdout, "<<<"); line+=3; } else { fprintf(stdout, "%c", toPrint); line++; } } else if(rules[i]<0x3400 || rules[i]>=0xa000) { fprintf(stdout, "\\u%04X", rules[i]); line+=6; } if(line>72) { fprintf(stdout, "\n"); line = 0; } } log_verbose("\n"); } */ static uint32_t testSwitch(tst_strcoll* func, void *collator, int opts, uint32_t strength, const UChar *first, const UChar *second, const char* msg, UBool error) { uint32_t diffs = 0; UCollationResult realResult; uint32_t realStrength; uint32_t sLen = u_strlen(first); uint32_t tLen = u_strlen(second); realResult = func(collator, opts, first, sLen, second, tLen); realStrength = probeStrength(func, collator, opts, first, sLen, second, tLen, realResult); if(strength == UCOL_IDENTICAL && realResult != UCOL_IDENTICAL) { logFailure(msg, "tailoring", first, sLen, second, tLen, realResult, realStrength, UCOL_EQUAL, strength, error); diffs++; } else if(realResult != UCOL_LESS || realStrength != strength) { logFailure(msg, "tailoring", first, sLen, second, tLen, realResult, realStrength, UCOL_LESS, strength, error); diffs++; } return diffs; } static void testAgainstUCA(UCollator *coll, UCollator *UCA, const char *refName, UBool error, UErrorCode *status) { const UChar *rules = NULL, *current = NULL; int32_t ruleLen = 0; uint32_t strength = 0; uint32_t chOffset = 0; uint32_t chLen = 0; uint32_t exOffset = 0; uint32_t exLen = 0; uint32_t prefixOffset = 0; uint32_t prefixLen = 0; /* uint32_t rExpsLen = 0; */ uint32_t firstLen = 0, secondLen = 0; UBool varT = FALSE; UBool top_ = TRUE; uint16_t specs = 0; UBool startOfRules = TRUE; UColTokenParser src; UColOptionSet opts; UChar first[256]; UChar second[256]; UChar *rulesCopy = NULL; uint32_t UCAdiff = 0; uint32_t Windiff = 1; UParseError parseError; src.opts = &opts; rules = ucol_getRules(coll, &ruleLen); /*printOutRules(rules);*/ if(U_SUCCESS(*status) && ruleLen > 0) { rulesCopy = (UChar *)malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar)); uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar)); src.current = src.source = rulesCopy; src.end = rulesCopy+ruleLen; src.extraCurrent = src.end; src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE; *first = *second = 0; while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,status)) != NULL) { strength = src.parsedToken.strength; chOffset = src.parsedToken.charsOffset; chLen = src.parsedToken.charsLen; exOffset = src.parsedToken.extensionOffset; exLen = src.parsedToken.extensionLen; prefixOffset = src.parsedToken.prefixOffset; prefixLen = src.parsedToken.prefixLen; specs = src.parsedToken.flags; startOfRules = FALSE; varT = (UBool)((specs & UCOL_TOK_VARIABLE_TOP) != 0); top_ = (UBool)((specs & UCOL_TOK_TOP) != 0); u_strncpy(second,rulesCopy+chOffset, chLen); second[chLen] = 0; secondLen = chLen; if(exLen > 0) { u_strncat(first, rulesCopy+exOffset, exLen); first[firstLen+exLen] = 0; firstLen += exLen; } if(strength != UCOL_TOK_RESET) { if((*first<0x3400 || *first>=0xa000) && (*second<0x3400 || *second>=0xa000)) { UCAdiff += testSwitch(&ucaTest, (void *)UCA, 0, strength, first, second, refName, error); /*Windiff += testSwitch(&winTest, (void *)lcid, 0, strength, first, second, "Win32");*/ } } firstLen = chLen; u_strcpy(first, second); } if(UCAdiff != 0 && Windiff != 0) { log_verbose("\n"); } if(UCAdiff == 0) { log_verbose("No immediate difference with %s!\n", refName); } if(Windiff == 0) { log_verbose("No immediate difference with Win32!\n"); } free(rulesCopy); } } /* * Takes two CEs (lead and continuation) and * compares them as CEs should be compared: * primary vs. primary, secondary vs. secondary * tertiary vs. tertiary */ static int32_t compareCEs(uint32_t s1, uint32_t s2, uint32_t t1, uint32_t t2) { uint32_t s = 0, t = 0; if(s1 == t1 && s2 == t2) { return 0; } s = (s1 & 0xFFFF0000)|((s2 & 0xFFFF0000)>>16); t = (t1 & 0xFFFF0000)|((t2 & 0xFFFF0000)>>16); if(s < t) { return -1; } else if(s > t) { return 1; } else { s = (s1 & 0x0000FF00) | (s2 & 0x0000FF00)>>8; t = (t1 & 0x0000FF00) | (t2 & 0x0000FF00)>>8; if(s < t) { return -1; } else if(s > t) { return 1; } else { s = (s1 & 0x000000FF)<<8 | (s2 & 0x000000FF); t = (t1 & 0x000000FF)<<8 | (t2 & 0x000000FF); if(s < t) { return -1; } else { return 1; } } } } typedef struct { uint32_t startCE; uint32_t startContCE; uint32_t limitCE; uint32_t limitContCE; } indirectBoundaries; /* these values are used for finding CE values for indirect positioning. */ /* Indirect positioning is a mechanism for allowing resets on symbolic */ /* values. It only works for resets and you cannot tailor indirect names */ /* An indirect name can define either an anchor point or a range. An */ /* anchor point behaves in exactly the same way as a code point in reset */ /* would, except that it cannot be tailored. A range (we currently only */ /* know for the [top] range will explicitly set the upper bound for */ /* generated CEs, thus allowing for better control over how many CEs can */ /* be squeezed between in the range without performance penalty. */ /* In that respect, we use [top] for tailoring of locales that use CJK */ /* characters. Other indirect values are currently a pure convenience, */ /* they can be used to assure that the CEs will be always positioned in */ /* the same place relative to a point with known properties (e.g. first */ /* primary ignorable). */ static indirectBoundaries ucolIndirectBoundaries[15]; static UBool indirectBoundariesSet = FALSE; static void setIndirectBoundaries(uint32_t indexR, uint32_t *start, uint32_t *end) { /* Set values for the top - TODO: once we have values for all the indirects, we are going */ /* to initalize here. */ ucolIndirectBoundaries[indexR].startCE = start[0]; ucolIndirectBoundaries[indexR].startContCE = start[1]; if(end) { ucolIndirectBoundaries[indexR].limitCE = end[0]; ucolIndirectBoundaries[indexR].limitContCE = end[1]; } else { ucolIndirectBoundaries[indexR].limitCE = 0; ucolIndirectBoundaries[indexR].limitContCE = 0; } } static void testCEs(UCollator *coll, UErrorCode *status) { const UChar *rules = NULL, *current = NULL; int32_t ruleLen = 0; uint32_t strength = 0; uint32_t maxStrength = UCOL_IDENTICAL; uint32_t baseCE, baseContCE, nextCE, nextContCE, currCE, currContCE; uint32_t lastCE; uint32_t lastContCE; int32_t result = 0; uint32_t chOffset = 0; uint32_t chLen = 0; uint32_t exOffset = 0; uint32_t exLen = 0; uint32_t prefixOffset = 0; uint32_t prefixLen = 0; uint32_t oldOffset = 0; /* uint32_t rExpsLen = 0; */ /* uint32_t firstLen = 0; */ uint16_t specs = 0; UBool varT = FALSE; UBool top_ = TRUE; UBool startOfRules = TRUE; UBool before = FALSE; UColTokenParser src; UColOptionSet opts; UParseError parseError; UChar *rulesCopy = NULL; collIterate c; UCollator *UCA = ucol_open("root", status); UCAConstants *consts = (UCAConstants *)((uint8_t *)UCA->image + UCA->image->UCAConsts); uint32_t UCOL_RESET_TOP_VALUE = consts->UCA_LAST_NON_VARIABLE[0], /*UCOL_RESET_TOP_CONT = consts->UCA_LAST_NON_VARIABLE[1], */ UCOL_NEXT_TOP_VALUE = consts->UCA_FIRST_IMPLICIT[0], UCOL_NEXT_TOP_CONT = consts->UCA_FIRST_IMPLICIT[1]; baseCE=baseContCE=nextCE=nextContCE=currCE=currContCE=lastCE=lastContCE = UCOL_NOT_FOUND; src.opts = &opts; rules = ucol_getRules(coll, &ruleLen); src.invUCA = ucol_initInverseUCA(status); if(indirectBoundariesSet == FALSE) { /* UCOL_RESET_TOP_VALUE */ setIndirectBoundaries(0, consts->UCA_LAST_NON_VARIABLE, consts->UCA_FIRST_IMPLICIT); /* UCOL_FIRST_PRIMARY_IGNORABLE */ setIndirectBoundaries(1, consts->UCA_FIRST_PRIMARY_IGNORABLE, 0); /* UCOL_LAST_PRIMARY_IGNORABLE */ setIndirectBoundaries(2, consts->UCA_LAST_PRIMARY_IGNORABLE, 0); /* UCOL_FIRST_SECONDARY_IGNORABLE */ setIndirectBoundaries(3, consts->UCA_FIRST_SECONDARY_IGNORABLE, 0); /* UCOL_LAST_SECONDARY_IGNORABLE */ setIndirectBoundaries(4, consts->UCA_LAST_SECONDARY_IGNORABLE, 0); /* UCOL_FIRST_TERTIARY_IGNORABLE */ setIndirectBoundaries(5, consts->UCA_FIRST_TERTIARY_IGNORABLE, 0); /* UCOL_LAST_TERTIARY_IGNORABLE */ setIndirectBoundaries(6, consts->UCA_LAST_TERTIARY_IGNORABLE, 0); /* UCOL_FIRST_VARIABLE */ setIndirectBoundaries(7, consts->UCA_FIRST_VARIABLE, 0); /* UCOL_LAST_VARIABLE */ setIndirectBoundaries(8, consts->UCA_LAST_VARIABLE, 0); /* UCOL_FIRST_NON_VARIABLE */ setIndirectBoundaries(9, consts->UCA_FIRST_NON_VARIABLE, 0); /* UCOL_LAST_NON_VARIABLE */ setIndirectBoundaries(10, consts->UCA_LAST_NON_VARIABLE, consts->UCA_FIRST_IMPLICIT); /* UCOL_FIRST_IMPLICIT */ setIndirectBoundaries(11, consts->UCA_FIRST_IMPLICIT, 0); /* UCOL_LAST_IMPLICIT */ setIndirectBoundaries(12, consts->UCA_LAST_IMPLICIT, consts->UCA_FIRST_TRAILING); /* UCOL_FIRST_TRAILING */ setIndirectBoundaries(13, consts->UCA_FIRST_TRAILING, 0); /* UCOL_LAST_TRAILING */ setIndirectBoundaries(14, consts->UCA_LAST_TRAILING, 0); ucolIndirectBoundaries[14].limitCE = (consts->UCA_PRIMARY_SPECIAL_MIN<<24); indirectBoundariesSet = TRUE; } if(U_SUCCESS(*status) && ruleLen > 0) { rulesCopy = (UChar *)malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar)); uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar)); src.current = src.source = rulesCopy; src.end = rulesCopy+ruleLen; src.extraCurrent = src.end; src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE; while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,status)) != NULL) { strength = src.parsedToken.strength; chOffset = src.parsedToken.charsOffset; chLen = src.parsedToken.charsLen; exOffset = src.parsedToken.extensionOffset; exLen = src.parsedToken.extensionLen; prefixOffset = src.parsedToken.prefixOffset; prefixLen = src.parsedToken.prefixLen; specs = src.parsedToken.flags; startOfRules = FALSE; varT = (UBool)((specs & UCOL_TOK_VARIABLE_TOP) != 0); top_ = (UBool)((specs & UCOL_TOK_TOP) != 0); uprv_init_collIterate(coll, rulesCopy+chOffset, chLen, &c); currCE = ucol_getNextCE(coll, &c, status); if(currCE == 0 && UCOL_ISTHAIPREVOWEL(*(rulesCopy+chOffset))) { log_verbose("Thai prevowel detected. Will pick next CE\n"); currCE = ucol_getNextCE(coll, &c, status); } currContCE = ucol_getNextCE(coll, &c, status); if(!isContinuation(currContCE)) { currContCE = 0; } /* we need to repack CEs here */ if(strength == UCOL_TOK_RESET) { before = (UBool)((specs & UCOL_TOK_BEFORE) != 0); if(top_ == TRUE) { int32_t index = src.parsedToken.indirectIndex; nextCE = baseCE = currCE = ucolIndirectBoundaries[index].startCE; nextContCE = baseContCE = currContCE = ucolIndirectBoundaries[index].startContCE; } else { nextCE = baseCE = currCE; nextContCE = baseContCE = currContCE; } maxStrength = UCOL_IDENTICAL; } else { if(strength < maxStrength) { maxStrength = strength; if(baseCE == UCOL_RESET_TOP_VALUE) { log_verbose("Resetting to [top]\n"); nextCE = UCOL_NEXT_TOP_VALUE; nextContCE = UCOL_NEXT_TOP_CONT; } else { result = ucol_inv_getNextCE(&src, baseCE & 0xFFFFFF3F, baseContCE, &nextCE, &nextContCE, maxStrength); } if(result < 0) { if(ucol_isTailored(coll, *(rulesCopy+oldOffset), status)) { log_verbose("Reset is tailored codepoint %04X, don't know how to continue, taking next test\n", *(rulesCopy+oldOffset)); return; } else { log_err("couldn't find the CE\n"); return; } } } currCE &= 0xFFFFFF3F; currContCE &= 0xFFFFFFBF; if(maxStrength == UCOL_IDENTICAL) { if(baseCE != currCE || baseContCE != currContCE) { log_err("current CE (initial strength UCOL_EQUAL)\n"); } } else { if(strength == UCOL_IDENTICAL) { if(lastCE != currCE || lastContCE != currContCE) { log_err("current CE (initial strength UCOL_EQUAL)\n"); } } else { if(compareCEs(currCE, currContCE, nextCE, nextContCE) > 0) { /*if(currCE > nextCE || (currCE == nextCE && currContCE >= nextContCE)) {*/ log_err("current CE is not less than base CE\n"); } if(!before) { if(compareCEs(currCE, currContCE, lastCE, lastContCE) < 0) { /*if(currCE < lastCE || (currCE == lastCE && currContCE <= lastContCE)) {*/ log_err("sequence of generated CEs is broken\n"); } } else { before = FALSE; if(compareCEs(currCE, currContCE, lastCE, lastContCE) > 0) { /*if(currCE < lastCE || (currCE == lastCE && currContCE <= lastContCE)) {*/ log_err("sequence of generated CEs is broken\n"); } } } } } oldOffset = chOffset; lastCE = currCE & 0xFFFFFF3F; lastContCE = currContCE & 0xFFFFFFBF; } free(rulesCopy); } ucol_close(UCA); } #if 0 /* these locales are now picked from index RB */ static const char* localesToTest[] = { "ar", "bg", "ca", "cs", "da", "el", "en_BE", "en_US_POSIX", "es", "et", "fi", "fr", "hi", "hr", "hu", "is", "iw", "ja", "ko", "lt", "lv", "mk", "mt", "nb", "nn", "nn_NO", "pl", "ro", "ru", "sh", "sk", "sl", "sq", "sr", "sv", "th", "tr", "uk", "vi", "zh", "zh_TW" }; #endif static const char* rulesToTest[] = { /* Funky fa rule */ "&\\u0622 < \\u0627 << \\u0671 < \\u0621", /*"& Z < p, P",*/ /* Cui Mins rules */ "&[top]image->jamoSpecial == TRUE) { log_err("%s has special JAMOs\n", locName); } ucol_setAttribute(coll, UCOL_CASE_FIRST, UCOL_OFF, &status); testCollator(coll, &status); testCEs(coll, &status); ucol_close(coll); } } } for(i = 0; iNFC, NORM_BUFFER_TEST_LEN, &status); nfdSize = unorm_normalize(comp, len, UNORM_NFD, 0, t[noCases]->NFD, NORM_BUFFER_TEST_LEN, &status); if(nfcSize != nfdSize || (uprv_memcmp(t[noCases]->NFC, t[noCases]->NFD, nfcSize * sizeof(UChar)) != 0) || (len != nfdSize || (uprv_memcmp(comp, t[noCases]->NFD, nfdSize * sizeof(UChar)) != 0))) { t[noCases]->u = u; if(len != nfdSize || (uprv_memcmp(comp, t[noCases]->NFD, nfdSize * sizeof(UChar)) != 0)) { u_strncpy(t[noCases]->NFC, comp, len); t[noCases]->NFC[len] = 0; } noCases++; t[noCases] = (tester *)malloc(sizeof(tester)); uprv_memset(t[noCases], 0, sizeof(tester)); } } for(u=0; u<(UChar32)noCases; u++) { if(!ucol_equal(coll, t[u]->NFC, -1, t[u]->NFD, -1)) { log_err("Failure: codePoint %05X fails TestComposeDecompose in the UCA\n", t[u]->u); doTest(coll, t[u]->NFC, t[u]->NFD, UCOL_EQUAL); } } /* for(u = 0; u < 0x30000; u++) { if(!(u&0xFFFF)) { log_verbose("%08X ", u); } uprv_memset(t[noCases], 0, sizeof(tester)); t[noCases]->u = u; len = 0; UTF_APPEND_CHAR_UNSAFE(comp, len, u); comp[len] = 0; nfcSize = unorm_normalize(comp, len, UNORM_NFC, 0, t[noCases]->NFC, NORM_BUFFER_TEST_LEN, &status); nfdSize = unorm_normalize(comp, len, UNORM_NFD, 0, t[noCases]->NFD, NORM_BUFFER_TEST_LEN, &status); doTest(coll, comp, t[noCases]->NFD, UCOL_EQUAL); doTest(coll, comp, t[noCases]->NFC, UCOL_EQUAL); } */ ucol_close(coll); log_verbose("Testing locales, number of cases = %i\n", noCases); for(i = 0; iNFD, u_strlen(t[u]->NFD), &status); for(u=0; u<(UChar32)noCases; u++) { if(!ucol_equal(coll, t[u]->NFC, -1, t[u]->NFD, -1)) { log_err("Failure: codePoint %05X fails TestComposeDecompose for locale %s\n", t[u]->u, cName); doTest(coll, t[u]->NFC, t[u]->NFD, UCOL_EQUAL); log_verbose("Testing NFC\n"); ucol_setText(iter, t[u]->NFC, u_strlen(t[u]->NFC), &status); backAndForth(iter); log_verbose("Testing NFD\n"); ucol_setText(iter, t[u]->NFD, u_strlen(t[u]->NFD), &status); backAndForth(iter); } } ucol_closeElements(iter); ucol_close(coll); } } for(u = 0; u <= (UChar32)noCases; u++) { free(t[u]); } free(t); } static void TestEmptyRule(void) { UErrorCode status = U_ZERO_ERROR; UChar rulez[] = { 0 }; UCollator *coll = ucol_openRules(rulez, 0, UCOL_OFF, UCOL_TERTIARY,NULL, &status); ucol_close(coll); } static void TestUCARules(void) { UErrorCode status = U_ZERO_ERROR; UChar b[256]; UChar *rules = b; uint32_t ruleLen = 0; UCollator *UCAfromRules = NULL; UCollator *coll = ucol_open("", &status); if(status == U_FILE_ACCESS_ERROR) { log_data_err("Is your data around?\n"); return; } else if(U_FAILURE(status)) { log_err("Error opening collator\n"); return; } ruleLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rules, 256); log_verbose("TestUCARules\n"); if(ruleLen > 256) { rules = (UChar *)malloc((ruleLen+1)*sizeof(UChar)); ruleLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rules, ruleLen); } log_verbose("Rules length is %d\n", ruleLen); UCAfromRules = ucol_openRules(rules, ruleLen, UCOL_OFF, UCOL_TERTIARY, NULL,&status); if(U_SUCCESS(status)) { ucol_close(UCAfromRules); } else { log_verbose("Unable to create a collator from UCARules!\n"); } /* u_unescape(blah, b, 256); ucol_getSortKey(coll, b, 1, res, 256); */ ucol_close(coll); if(rules != b) { free(rules); } } /* Pinyin tonal order */ /* A < .. (\u0101) < .. (\u00e1) < .. (\u01ce) < .. (\u00e0) (w/macron)< (w/acute)< (w/caron)< (w/grave) E < .. (\u0113) < .. (\u00e9) < .. (\u011b) < .. (\u00e8) I < .. (\u012b) < .. (\u00ed) < .. (\u01d0) < .. (\u00ec) O < .. (\u014d) < .. (\u00f3) < .. (\u01d2) < .. (\u00f2) U < .. (\u016b) < .. (\u00fa) < .. (\u01d4) < .. (\u00f9) < .. (\u01d6) < .. (\u01d8) < .. (\u01da) < .. (\u01dc) < .. (\u00fc) However, in testing we got the following order: A < .. (\u00e1) < .. (\u00e0) < .. (\u01ce) < .. (\u0101) (w/acute)< (w/grave)< (w/caron)< (w/macron) E < .. (\u00e9) < .. (\u00e8) < .. (\u00ea) < .. (\u011b) < .. (\u0113) I < .. (\u00ed) < .. (\u00ec) < .. (\u01d0) < .. (\u012b) O < .. (\u00f3) < .. (\u00f2) < .. (\u01d2) < .. (\u014d) U < .. (\u00fa) < .. (\u00f9) < .. (\u01d4) < .. (\u00fc) < .. (\u01d8) < .. (\u01dc) < .. (\u01da) < .. (\u01d6) < .. (\u016b) */ static void TestBefore(void) { const static char *data[] = { "\\u0101", "\\u00e1", "\\u01ce", "\\u00e0", "A", "\\u0113", "\\u00e9", "\\u011b", "\\u00e8", "E", "\\u012b", "\\u00ed", "\\u01d0", "\\u00ec", "I", "\\u014d", "\\u00f3", "\\u01d2", "\\u00f2", "O", "\\u016b", "\\u00fa", "\\u01d4", "\\u00f9", "U", "\\u01d6", "\\u01d8", "\\u01da", "\\u01dc", "\\u00fc" }; genericRulesStarter( "&[before 1]a<\\u0101<\\u00e1<\\u01ce<\\u00e0" "&[before 1]e<\\u0113<\\u00e9<\\u011b<\\u00e8" "&[before 1]i<\\u012b<\\u00ed<\\u01d0<\\u00ec" "&[before 1]o<\\u014d<\\u00f3<\\u01d2<\\u00f2" "&[before 1]u<\\u016b<\\u00fa<\\u01d4<\\u00f9" "&u<\\u01d6<\\u01d8<\\u01da<\\u01dc<\\u00fc", data, sizeof(data)/sizeof(data[0])); } #if 0 /* superceded by TestBeforePinyin */ static void TestJ784(void) { const static char *data[] = { "A", "\\u0101", "\\u00e1", "\\u01ce", "\\u00e0", "E", "\\u0113", "\\u00e9", "\\u011b", "\\u00e8", "I", "\\u012b", "\\u00ed", "\\u01d0", "\\u00ec", "O", "\\u014d", "\\u00f3", "\\u01d2", "\\u00f2", "U", "\\u016b", "\\u00fa", "\\u01d4", "\\u00f9", "\\u00fc", "\\u01d6", "\\u01d8", "\\u01da", "\\u01dc" }; genericLocaleStarter("zh", data, sizeof(data)/sizeof(data[0])); } #endif #if 0 /* superceded by the changes to the lv locale */ static void TestJ831(void) { const static char *data[] = { "I", "i", "Y", "y" }; genericLocaleStarter("lv", data, sizeof(data)/sizeof(data[0])); } #endif static void TestJ815(void) { const static char *data[] = { "aa", "Aa", "ab", "Ab", "ad", "Ad", "ae", "Ae", "\\u00e6", "\\u00c6", "af", "Af", "b", "B" }; genericLocaleStarter("fr", data, sizeof(data)/sizeof(data[0])); genericRulesStarter("[backwards 2]&A<<\\u00e6/e<<<\\u00c6/E", data, sizeof(data)/sizeof(data[0])); } /* "& a < b < c < d& r < c", "& a < b < d& r < c", "& a < b < c < d& c < m", "& a < b < c < m < d", "& a < b < c < d& a < m", "& a < m < b < c < d", "& a <<< b << c < d& a < m", "& a <<< b << c < m < d", "& a < b < c < d& [before 1] c < m", "& a < b < m < c < d", "& a < b <<< c << d <<< e& [before 3] e <<< x", "& a < b <<< c << d <<< x <<< e", "& a < b <<< c << d <<< e& [before 2] e <<< x", "& a < b <<< c <<< x << d <<< e", "& a < b <<< c << d <<< e& [before 1] e <<< x", "& a <<< x < b <<< c << d <<< e", "& a < b <<< c << d <<< e <<< f < g& [before 1] g < x", "& a < b <<< c << d <<< e <<< f < x < g", */ static void TestRedundantRules(void) { int32_t i; struct { const char *rules; const char *expectedRules; const char *testdata[8]; uint32_t testdatalen; } tests[] = { /* this test conflicts with positioning of CODAN placeholder */ /*{ "& a <<< b <<< c << d <<< e& [before 1] e <<< x", "&\\u2089<<image)->jamoSpecial = TRUE; /* don't try this at home */ genericOrderingTest(coll, koreanData, sizeof(koreanData)/sizeof(koreanData[0])); ucol_close(coll); log_verbose("Using ko__LOTUS locale\n"); genericLocaleStarter("ko__LOTUS", koreanData, sizeof(koreanData)/sizeof(koreanData[0])); } static void TestCompressOverlap(void) { UChar secstr[150]; UChar tertstr[150]; UErrorCode status = U_ZERO_ERROR; UCollator *coll; char result[200]; uint32_t resultlen; int count = 0; char *tempptr; coll = ucol_open("", &status); if (U_FAILURE(status)) { log_err("Collator can't be created\n"); return; } while (count < 149) { secstr[count] = 0x0020; /* [06, 05, 05] */ tertstr[count] = 0x0020; count ++; } /* top down compression ----------------------------------- */ secstr[count] = 0x0332; /* [, 87, 05] */ tertstr[count] = 0x3000; /* [06, 05, 07] */ /* no compression secstr should have 150 secondary bytes, tertstr should have 150 tertiary bytes. with correct overlapping compression, secstr should have 4 secondary bytes, tertstr should have > 2 tertiary bytes */ resultlen = ucol_getSortKey(coll, secstr, 150, (uint8_t *)result, 250); tempptr = uprv_strchr(result, 1) + 1; while (*(tempptr + 1) != 1) { /* the last secondary collation element is not checked since it is not part of the compression */ if (*tempptr < UCOL_COMMON_TOP2 - UCOL_TOP_COUNT2) { log_err("Secondary compression overlapped\n"); } tempptr ++; } /* tertiary top/bottom/common for en_US is similar to the secondary top/bottom/common */ resultlen = ucol_getSortKey(coll, tertstr, 150, (uint8_t *)result, 250); tempptr = uprv_strrchr(result, 1) + 1; while (*(tempptr + 1) != 0) { /* the last secondary collation element is not checked since it is not part of the compression */ if (*tempptr < coll->tertiaryTop - coll->tertiaryTopCount) { log_err("Tertiary compression overlapped\n"); } tempptr ++; } /* bottom up compression ------------------------------------- */ secstr[count] = 0; tertstr[count] = 0; resultlen = ucol_getSortKey(coll, secstr, 150, (uint8_t *)result, 250); tempptr = uprv_strchr(result, 1) + 1; while (*(tempptr + 1) != 1) { /* the last secondary collation element is not checked since it is not part of the compression */ if (*tempptr > UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2) { log_err("Secondary compression overlapped\n"); } tempptr ++; } /* tertiary top/bottom/common for en_US is similar to the secondary top/bottom/common */ resultlen = ucol_getSortKey(coll, tertstr, 150, (uint8_t *)result, 250); tempptr = uprv_strrchr(result, 1) + 1; while (*(tempptr + 1) != 0) { /* the last secondary collation element is not checked since it is not part of the compression */ if (*tempptr > coll->tertiaryBottom + coll->tertiaryBottomCount) { log_err("Tertiary compression overlapped\n"); } tempptr ++; } ucol_close(coll); } static void TestCyrillicTailoring(void) { static const char *test[] = { "\\u0410b", "\\u0410\\u0306a", "\\u04d0A" }; /* Russian overrides contractions, so this test is not valid anymore */ /*genericLocaleStarter("ru", test, 3);*/ genericLocaleStarter("root", test, 3); genericRulesStarter("&\\u0410 = \\u0410", test, 3); genericRulesStarter("&Z < \\u0410", test, 3); genericRulesStarter("&\\u0410 = \\u0410 < \\u04d0", test, 3); genericRulesStarter("&Z < \\u0410 < \\u04d0", test, 3); genericRulesStarter("&\\u0410 = \\u0410 < \\u0410\\u0301", test, 3); genericRulesStarter("&Z < \\u0410 < \\u0410\\u0301", test, 3); } static void TestSuppressContractions(void) { static const char *testNoCont2[] = { "\\u0410\\u0302a", "\\u0410\\u0306b", "\\u0410c" }; static const char *testNoCont[] = { "a\\u0410", "A\\u0410\\u0306", "\\uFF21\\u0410\\u0302" }; genericRulesStarter("[suppressContractions [\\u0400-\\u047f]]", testNoCont, 3); genericRulesStarter("[suppressContractions [\\u0400-\\u047f]]", testNoCont2, 3); } static void TestContraction(void) { const static char *testrules[] = { "&A = AB / B", "&A = A\\u0306/\\u0306", "&c = ch / h" }; const static UChar testdata[][2] = { {0x0041 /* 'A' */, 0x0042 /* 'B' */}, {0x0041 /* 'A' */, 0x0306 /* combining breve */}, {0x0063 /* 'c' */, 0x0068 /* 'h' */} }; const static UChar testdata2[][2] = { {0x0063 /* 'c' */, 0x0067 /* 'g' */}, {0x0063 /* 'c' */, 0x0068 /* 'h' */}, {0x0063 /* 'c' */, 0x006C /* 'l' */} }; const static char *testrules3[] = { "&z < xyz &xyzw << B", "&z < xyz &xyz << B / w", "&z < ch &achm << B", "&z < ch &a << B / chm", "&\\ud800\\udc00w << B", "&\\ud800\\udc00 << B / w", "&a\\ud800\\udc00m << B", "&a << B / \\ud800\\udc00m", }; UErrorCode status = U_ZERO_ERROR; UCollator *coll; UChar rule[256] = {0}; uint32_t rlen = 0; int i; for (i = 0; i < sizeof(testrules) / sizeof(testrules[0]); i ++) { UCollationElements *iter1; int j = 0; log_verbose("Rule %s for testing\n", testrules[i]); rlen = u_unescape(testrules[i], rule, 32); coll = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status); if (U_FAILURE(status)) { log_err("Collator creation failed %s\n", testrules[i]); return; } iter1 = ucol_openElements(coll, testdata[i], 2, &status); if (U_FAILURE(status)) { log_err("Collation iterator creation failed\n"); return; } while (j < 2) { UCollationElements *iter2 = ucol_openElements(coll, &(testdata[i][j]), 1, &status); uint32_t ce; if (U_FAILURE(status)) { log_err("Collation iterator creation failed\n"); return; } ce = ucol_next(iter2, &status); while (ce != UCOL_NULLORDER) { if ((uint32_t)ucol_next(iter1, &status) != ce) { log_err("Collation elements in contraction split does not match\n"); return; } ce = ucol_next(iter2, &status); } j ++; ucol_closeElements(iter2); } if (ucol_next(iter1, &status) != UCOL_NULLORDER) { log_err("Collation elements not exhausted\n"); return; } ucol_closeElements(iter1); ucol_close(coll); } rlen = u_unescape("& a < b < c < ch < d & c = ch / h", rule, 256); coll = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status); if (ucol_strcoll(coll, testdata2[0], 2, testdata2[1], 2) != UCOL_LESS) { log_err("Expected \\u%04x\\u%04x < \\u%04x\\u%04x\n", testdata2[0][0], testdata2[0][1], testdata2[1][0], testdata2[1][1]); return; } if (ucol_strcoll(coll, testdata2[1], 2, testdata2[2], 2) != UCOL_LESS) { log_err("Expected \\u%04x\\u%04x < \\u%04x\\u%04x\n", testdata2[1][0], testdata2[1][1], testdata2[2][0], testdata2[2][1]); return; } ucol_close(coll); for (i = 0; i < sizeof(testrules3) / sizeof(testrules3[0]); i += 2) { UCollator *coll1, *coll2; UCollationElements *iter1, *iter2; UChar ch = 0x0042 /* 'B' */; uint32_t ce; rlen = u_unescape(testrules3[i], rule, 32); coll1 = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status); rlen = u_unescape(testrules3[i + 1], rule, 32); coll2 = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status); if (U_FAILURE(status)) { log_err("Collator creation failed %s\n", testrules[i]); return; } iter1 = ucol_openElements(coll1, &ch, 1, &status); iter2 = ucol_openElements(coll2, &ch, 1, &status); if (U_FAILURE(status)) { log_err("Collation iterator creation failed\n"); return; } ce = ucol_next(iter1, &status); if (U_FAILURE(status)) { log_err("Retrieving ces failed\n"); return; } while (ce != UCOL_NULLORDER) { if (ce != (uint32_t)ucol_next(iter2, &status)) { log_err("CEs does not match\n"); return; } ce = ucol_next(iter1, &status); if (U_FAILURE(status)) { log_err("Retrieving ces failed\n"); return; } } if (ucol_next(iter2, &status) != UCOL_NULLORDER) { log_err("CEs not exhausted\n"); return; } ucol_closeElements(iter1); ucol_closeElements(iter2); ucol_close(coll1); ucol_close(coll2); } } static void TestExpansion(void) { const static char *testrules[] = { "&J << K / B & K << M", "&J << K / B << M" }; const static UChar testdata[][3] = { {0x004A /*'J'*/, 0x0041 /*'A'*/, 0}, {0x004D /*'M'*/, 0x0041 /*'A'*/, 0}, {0x004B /*'K'*/, 0x0041 /*'A'*/, 0}, {0x004B /*'K'*/, 0x0043 /*'C'*/, 0}, {0x004A /*'J'*/, 0x0043 /*'C'*/, 0}, {0x004D /*'M'*/, 0x0043 /*'C'*/, 0} }; UErrorCode status = U_ZERO_ERROR; UCollator *coll; UChar rule[256] = {0}; uint32_t rlen = 0; int i; for (i = 0; i < sizeof(testrules) / sizeof(testrules[0]); i ++) { int j = 0; log_verbose("Rule %s for testing\n", testrules[i]); rlen = u_unescape(testrules[i], rule, 32); coll = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status); if (U_FAILURE(status)) { log_err("Collator creation failed %s\n", testrules[i]); return; } for (j = 0; j < 5; j ++) { doTest(coll, testdata[j], testdata[j + 1], UCOL_LESS); } ucol_close(coll); } } #if 0 /* this test tests the current limitations of the engine */ /* it always fail, so it is disabled by default */ static void TestLimitations(void) { /* recursive expansions */ { static const char *rule = "&a=b/c&d=c/e"; static const char *tlimit01[] = {"add","b","adf"}; static const char *tlimit02[] = {"aa","b","af"}; log_verbose("recursive expansions\n"); genericRulesStarter(rule, tlimit01, sizeof(tlimit01)/sizeof(tlimit01[0])); genericRulesStarter(rule, tlimit02, sizeof(tlimit02)/sizeof(tlimit02[0])); } /* contractions spanning expansions */ { static const char *rule = "&a<< 0*/ log_verbose("Slide variable top over UCARules\n"); rulesLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rulesCopy, 0); rulesCopy = (UChar *)malloc((rulesLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar)); rulesLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rulesCopy, rulesLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE); if(U_SUCCESS(status) && rulesLen > 0) { ucol_setAttribute(coll, UCOL_ALTERNATE_HANDLING, UCOL_SHIFTED, &status); src.current = src.source = rulesCopy; src.end = rulesCopy+rulesLen; src.extraCurrent = src.end; src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE; while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,&status)) != NULL) { strength = src.parsedToken.strength; chOffset = src.parsedToken.charsOffset; chLen = src.parsedToken.charsLen; exOffset = src.parsedToken.extensionOffset; exLen = src.parsedToken.extensionLen; prefixOffset = src.parsedToken.prefixOffset; prefixLen = src.parsedToken.prefixLen; specs = src.parsedToken.flags; startOfRules = FALSE; if(0) { log_verbose("%04X %d ", *(rulesCopy+chOffset), chLen); } if(strength == UCOL_PRIMARY) { status = U_ZERO_ERROR; varTopOriginal = ucol_getVariableTop(coll, &status); varTop1 = ucol_setVariableTop(coll, rulesCopy+oldChOffset, oldChLen, &status); if(U_FAILURE(status)) { char buffer[256]; char *buf = buffer; uint32_t i = 0, j; uint32_t CE = UCOL_NO_MORE_CES; /* before we start screaming, let's see if there is a problem with the rules */ collIterate s; uprv_init_collIterate(coll, rulesCopy+oldChOffset, oldChLen, &s); CE = ucol_getNextCE(coll, &s, &status); for(i = 0; i < oldChLen; i++) { j = sprintf(buf, "%04X ", *(rulesCopy+oldChOffset+i)); buf += j; } if(status == U_PRIMARY_TOO_LONG_ERROR) { log_verbose("= Expected failure for %s =", buffer); } else { if(s.pos == s.endp) { log_err("Unexpected failure setting variable top at offset %d. Error %s. Codepoints: %s\n", oldChOffset, u_errorName(status), buffer); } else { log_verbose("There is a goofy contraction in UCA rules that does not appear in the fractional UCA. Codepoints: %s\n", buffer); } } } varTop2 = ucol_getVariableTop(coll, &status); if((varTop1 & 0xFFFF0000) != (varTop2 & 0xFFFF0000)) { log_err("cannot retrieve set varTop value!\n"); continue; } if((varTop1 & 0xFFFF0000) > 0 && oldExLen == 0) { u_strncpy(first, rulesCopy+oldChOffset, oldChLen); u_strncpy(first+oldChLen, rulesCopy+chOffset, chLen); u_strncpy(first+oldChLen+chLen, rulesCopy+oldChOffset, oldChLen); first[2*oldChLen+chLen] = 0; if(oldExLen == 0) { u_strncpy(second, rulesCopy+chOffset, chLen); second[chLen] = 0; } else { /* This is skipped momentarily, but should work once UCARules are fully UCA conformant */ u_strncpy(second, rulesCopy+oldExOffset, oldExLen); u_strncpy(second+oldChLen, rulesCopy+chOffset, chLen); u_strncpy(second+oldChLen+chLen, rulesCopy+oldExOffset, oldExLen); second[2*oldExLen+chLen] = 0; } result = ucol_strcoll(coll, first, -1, second, -1); if(result == UCOL_EQUAL) { doTest(coll, first, second, UCOL_EQUAL); } else { log_verbose("Suspicious strcoll result for %04X and %04X\n", *(rulesCopy+oldChOffset), *(rulesCopy+chOffset)); } } } if(strength != UCOL_TOK_RESET) { oldChOffset = chOffset; oldChLen = chLen; oldExOffset = exOffset; oldExLen = exLen; } } status = U_ZERO_ERROR; } else { log_err("Unexpected failure getting rules %s\n", u_errorName(status)); return; } if (U_FAILURE(status)) { log_err("Error parsing rules %s\n", u_errorName(status)); return; } status = U_ZERO_ERROR; } QUICK = myQ; log_verbose("Testing setting variable top to contractions\n"); { /* uint32_t tailoredCE = UCOL_NOT_FOUND; */ /*UChar *conts = (UChar *)((uint8_t *)coll->image + coll->image->UCAConsts+sizeof(UCAConstants));*/ UChar *conts = (UChar *)((uint8_t *)coll->image + coll->image->contractionUCACombos); while(*conts != 0) { if(*(conts+2) == 0) { varTop1 = ucol_setVariableTop(coll, conts, -1, &status); } else { varTop1 = ucol_setVariableTop(coll, conts, 3, &status); } if(U_FAILURE(status)) { log_err("Couldn't set variable top to a contraction %04X %04X %04X\n", *conts, *(conts+1), *(conts+2)); status = U_ZERO_ERROR; } conts+=3; } status = U_ZERO_ERROR; first[0] = 0x0040; first[1] = 0x0050; first[2] = 0x0000; ucol_setVariableTop(coll, first, -1, &status); if(U_SUCCESS(status)) { log_err("Invalid contraction succeded in setting variable top!\n"); } } log_verbose("Test restoring variable top\n"); status = U_ZERO_ERROR; ucol_restoreVariableTop(coll, varTopOriginal, &status); if(varTopOriginal != ucol_getVariableTop(coll, &status)) { log_err("Couldn't restore old variable top\n"); } log_verbose("Testing calling with error set\n"); status = U_INTERNAL_PROGRAM_ERROR; varTop1 = ucol_setVariableTop(coll, first, 1, &status); varTop2 = ucol_getVariableTop(coll, &status); ucol_restoreVariableTop(coll, varTop2, &status); varTop1 = ucol_setVariableTop(NULL, first, 1, &status); varTop2 = ucol_getVariableTop(NULL, &status); ucol_restoreVariableTop(NULL, varTop2, &status); if(status != U_INTERNAL_PROGRAM_ERROR) { log_err("Bad reaction to passed error!\n"); } free(rulesCopy); ucol_close(coll); } else { log_data_err("Couldn't open UCA collator\n"); } } static void TestNonChars(void) { static const char *test[] = { "\\u0000", "\\uFFFE", "\\uFFFF", "\\U0001FFFE", "\\U0001FFFF", "\\U0002FFFE", "\\U0002FFFF", "\\U0003FFFE", "\\U0003FFFF", "\\U0004FFFE", "\\U0004FFFF", "\\U0005FFFE", "\\U0005FFFF", "\\U0006FFFE", "\\U0006FFFF", "\\U0007FFFE", "\\U0007FFFF", "\\U0008FFFE", "\\U0008FFFF", "\\U0009FFFE", "\\U0009FFFF", "\\U000AFFFE", "\\U000AFFFF", "\\U000BFFFE", "\\U000BFFFF", "\\U000CFFFE", "\\U000CFFFF", "\\U000DFFFE", "\\U000DFFFF", "\\U000EFFFE", "\\U000EFFFF", "\\U000FFFFE", "\\U000FFFFF", "\\U0010FFFE", "\\U0010FFFF" }; UErrorCode status = U_ZERO_ERROR; UCollator *coll = ucol_open("en_US", &status); log_verbose("Test non characters\n"); if(U_SUCCESS(status)) { genericOrderingTestWithResult(coll, test, 35, UCOL_EQUAL); } else { log_err("Unable to open collator\n"); } ucol_close(coll); } static void TestExtremeCompression(void) { static char *test[4]; int32_t j = 0, i = 0; for(i = 0; i<4; i++) { test[i] = (char *)malloc(2048*sizeof(char)); } for(j = 20; j < 500; j++) { for(i = 0; i<4; i++) { uprv_memset(test[i], 'a', (j-1)*sizeof(char)); test[i][j-1] = (char)('a'+i); test[i][j] = 0; } genericLocaleStarter("en_US", (const char **)test, 4); } for(i = 0; i<4; i++) { free(test[i]); } } #if 0 static void TestExtremeCompression(void) { static char *test[4]; int32_t j = 0, i = 0; UErrorCode status = U_ZERO_ERROR; UCollator *coll = ucol_open("en_US", status); for(i = 0; i<4; i++) { test[i] = (char *)malloc(2048*sizeof(char)); } for(j = 10; j < 2048; j++) { for(i = 0; i<4; i++) { uprv_memset(test[i], 'a', (j-2)*sizeof(char)); test[i][j-1] = (char)('a'+i); test[i][j] = 0; } } genericLocaleStarter("en_US", (const char **)test, 4); for(j = 10; j < 2048; j++) { for(i = 0; i<1; i++) { uprv_memset(test[i], 'a', (j-1)*sizeof(char)); test[i][j] = 0; } } for(i = 0; i<4; i++) { free(test[i]); } } #endif static void TestSurrogates(void) { static const char *test[] = { "z","\\ud900\\udc25", "\\ud805\\udc50", "\\ud800\\udc00y", "\\ud800\\udc00r", "\\ud800\\udc00f", "\\ud800\\udc00", "\\ud800\\udc00c", "\\ud800\\udc00b", "\\ud800\\udc00fa", "\\ud800\\udc00fb", "\\ud800\\udc00a", "c", "b" }; static const char *rule = "&z < \\ud900\\udc25 < \\ud805\\udc50" "< \\ud800\\udc00y < \\ud800\\udc00r" "< \\ud800\\udc00f << \\ud800\\udc00" "< \\ud800\\udc00fa << \\ud800\\udc00fb" "< \\ud800\\udc00a < c < b" ; genericRulesStarter(rule, test, 14); } /* This is a test for prefix implementation, used by JIS X 4061 collation rules */ static void TestPrefix(void) { uint32_t i; static struct { const char *rules; const char *data[50]; const uint32_t len; } tests[] = { { "&z <<< z|a", {"zz", "za"}, 2 }, { "&z <<< z| a", {"zz", "za"}, 2 }, { "[strength I]" "&a=\\ud900\\udc25" "&z<<<\\ud900\\udc25|a", {"aa", "az", "\\ud900\\udc25z", "\\ud900\\udc25a", "zz"}, 4 }, }; for(i = 0; i<(sizeof(tests)/sizeof(tests[0])); i++) { genericRulesStarter(tests[i].rules, tests[i].data, tests[i].len); } } /* This test uses data suplied by Masashiko Maedera to test the implementation */ /* JIS X 4061 collation order implementation */ static void TestNewJapanese(void) { static const char *test1[] = { "\\u30b7\\u30e3\\u30fc\\u30ec", "\\u30b7\\u30e3\\u30a4", "\\u30b7\\u30e4\\u30a3", "\\u30b7\\u30e3\\u30ec", "\\u3061\\u3087\\u3053", "\\u3061\\u3088\\u3053", "\\u30c1\\u30e7\\u30b3\\u30ec\\u30fc\\u30c8", "\\u3066\\u30fc\\u305f", "\\u30c6\\u30fc\\u30bf", "\\u30c6\\u30a7\\u30bf", "\\u3066\\u3048\\u305f", "\\u3067\\u30fc\\u305f", "\\u30c7\\u30fc\\u30bf", "\\u30c7\\u30a7\\u30bf", "\\u3067\\u3048\\u305f", "\\u3066\\u30fc\\u305f\\u30fc", "\\u30c6\\u30fc\\u30bf\\u30a1", "\\u30c6\\u30a7\\u30bf\\u30fc", "\\u3066\\u3047\\u305f\\u3041", "\\u3066\\u3048\\u305f\\u30fc", "\\u3067\\u30fc\\u305f\\u30fc", "\\u30c7\\u30fc\\u30bf\\u30a1", "\\u3067\\u30a7\\u305f\\u30a1", "\\u30c7\\u3047\\u30bf\\u3041", "\\u30c7\\u30a8\\u30bf\\u30a2", "\\u3072\\u3086", "\\u3073\\u3085\\u3042", "\\u3074\\u3085\\u3042", "\\u3073\\u3085\\u3042\\u30fc", "\\u30d3\\u30e5\\u30a2\\u30fc", "\\u3074\\u3085\\u3042\\u30fc", "\\u30d4\\u30e5\\u30a2\\u30fc", "\\u30d2\\u30e5\\u30a6", "\\u30d2\\u30e6\\u30a6", "\\u30d4\\u30e5\\u30a6\\u30a2", "\\u3073\\u3085\\u30fc\\u3042\\u30fc", "\\u30d3\\u30e5\\u30fc\\u30a2\\u30fc", "\\u30d3\\u30e5\\u30a6\\u30a2\\u30fc", "\\u3072\\u3085\\u3093", "\\u3074\\u3085\\u3093", "\\u3075\\u30fc\\u308a", "\\u30d5\\u30fc\\u30ea", "\\u3075\\u3045\\u308a", "\\u3075\\u30a5\\u308a", "\\u3075\\u30a5\\u30ea", "\\u30d5\\u30a6\\u30ea", "\\u3076\\u30fc\\u308a", "\\u30d6\\u30fc\\u30ea", "\\u3076\\u3045\\u308a", "\\u30d6\\u30a5\\u308a", "\\u3077\\u3046\\u308a", "\\u30d7\\u30a6\\u30ea", "\\u3075\\u30fc\\u308a\\u30fc", "\\u30d5\\u30a5\\u30ea\\u30fc", "\\u3075\\u30a5\\u308a\\u30a3", "\\u30d5\\u3045\\u308a\\u3043", "\\u30d5\\u30a6\\u30ea\\u30fc", "\\u3075\\u3046\\u308a\\u3043", "\\u30d6\\u30a6\\u30ea\\u30a4", "\\u3077\\u30fc\\u308a\\u30fc", "\\u3077\\u30a5\\u308a\\u30a4", "\\u3077\\u3046\\u308a\\u30fc", "\\u30d7\\u30a6\\u30ea\\u30a4", "\\u30d5\\u30fd", "\\u3075\\u309e", "\\u3076\\u309d", "\\u3076\\u3075", "\\u3076\\u30d5", "\\u30d6\\u3075", "\\u30d6\\u30d5", "\\u3076\\u309e", "\\u3076\\u3077", "\\u30d6\\u3077", "\\u3077\\u309d", "\\u30d7\\u30fd", "\\u3077\\u3075", }; static const char *test2[] = { "\\u306f\\u309d", /* H\\u309d */ /*"\\u30cf\\u30fd",*/ /* K\\u30fd */ "\\u306f\\u306f", /* HH */ "\\u306f\\u30cf", /* HK */ "\\u30cf\\u30cf", /* KK */ "\\u306f\\u309e", /* H\\u309e */ "\\u30cf\\u30fe", /* K\\u30fe */ "\\u306f\\u3070", /* HH\\u309b */ "\\u30cf\\u30d0", /* KK\\u309b */ "\\u306f\\u3071", /* HH\\u309c */ "\\u30cf\\u3071", /* KH\\u309c */ "\\u30cf\\u30d1", /* KK\\u309c */ "\\u3070\\u309d", /* H\\u309b\\u309d */ "\\u30d0\\u30fd", /* K\\u309b\\u30fd */ "\\u3070\\u306f", /* H\\u309bH */ "\\u30d0\\u30cf", /* K\\u309bK */ "\\u3070\\u309e", /* H\\u309b\\u309e */ "\\u30d0\\u30fe", /* K\\u309b\\u30fe */ "\\u3070\\u3070", /* H\\u309bH\\u309b */ "\\u30d0\\u3070", /* K\\u309bH\\u309b */ "\\u30d0\\u30d0", /* K\\u309bK\\u309b */ "\\u3070\\u3071", /* H\\u309bH\\u309c */ "\\u30d0\\u30d1", /* K\\u309bK\\u309c */ "\\u3071\\u309d", /* H\\u309c\\u309d */ "\\u30d1\\u30fd", /* K\\u309c\\u30fd */ "\\u3071\\u306f", /* H\\u309cH */ "\\u30d1\\u30cf", /* K\\u309cK */ "\\u3071\\u3070", /* H\\u309cH\\u309b */ "\\u3071\\u30d0", /* H\\u309cK\\u309b */ "\\u30d1\\u30d0", /* K\\u309cK\\u309b */ "\\u3071\\u3071", /* H\\u309cH\\u309c */ "\\u30d1\\u30d1", /* K\\u309cK\\u309c */ }; /* static const char *test3[] = { "\\u221er\\u221e", "\\u221eR#", "\\u221et\\u221e", "#r\\u221e", "#R#", "#t%", "#T%", "8t\\u221e", "8T\\u221e", "8t#", "8T#", "8t%", "8T%", "8t8", "8T8", "\\u03c9r\\u221e", "\\u03a9R%", "rr\\u221e", "rR\\u221e", "Rr\\u221e", "RR\\u221e", "RT%", "rt8", "tr\\u221e", "tr8", "TR8", "tt8", "\\u30b7\\u30e3\\u30fc\\u30ec", }; */ static const UColAttribute att[] = { UCOL_STRENGTH }; static const UColAttributeValue val[] = { UCOL_QUATERNARY }; static const UColAttribute attShifted[] = { UCOL_STRENGTH, UCOL_ALTERNATE_HANDLING}; static const UColAttributeValue valShifted[] = { UCOL_QUATERNARY, UCOL_SHIFTED }; genericLocaleStarterWithOptions("ja", test1, sizeof(test1)/sizeof(test1[0]), att, val, 1); genericLocaleStarterWithOptions("ja", test2, sizeof(test2)/sizeof(test2[0]), att, val, 1); /*genericLocaleStarter("ja", test3, sizeof(test3)/sizeof(test3[0]));*/ genericLocaleStarterWithOptions("ja", test1, sizeof(test1)/sizeof(test1[0]), attShifted, valShifted, 2); genericLocaleStarterWithOptions("ja", test2, sizeof(test2)/sizeof(test2[0]), attShifted, valShifted, 2); } static void TestStrCollIdenticalPrefix(void) { const char* rule = "&\\ud9b0\\udc70=\\ud9b0\\udc71"; const char* test[] = { "ab\\ud9b0\\udc70", "ab\\ud9b0\\udc71" }; genericRulesTestWithResult(rule, test, sizeof(test)/sizeof(test[0]), UCOL_EQUAL); } /* Contractions should have all their canonically equivalent */ /* strings included */ static void TestContractionClosure(void) { static struct { const char *rules; const char *data[50]; const uint32_t len; } tests[] = { { "&b=\\u00e4\\u00e4", { "b", "\\u00e4\\u00e4", "a\\u0308a\\u0308", "\\u00e4a\\u0308", "a\\u0308\\u00e4" }, 5}, { "&b=\\u00C5", { "b", "\\u00C5", "A\\u030A", "\\u212B" }, 4}, }; uint32_t i; for(i = 0; i<(sizeof(tests)/sizeof(tests[0])); i++) { genericRulesTestWithResult(tests[i].rules, tests[i].data, tests[i].len, UCOL_EQUAL); } } /* This tests also fails*/ static void TestBeforePrefixFailure(void) { static struct { const char *rules; const char *data[50]; const uint32_t len; } tests[] = { { "&g <<< a" "&[before 3]\\uff41 <<< x", {"x", "\\uff41"}, 2 }, { "&\\u30A7=\\u30A7=\\u3047=\\uff6a" "&\\u30A8=\\u30A8=\\u3048=\\uff74" "&[before 3]\\u30a7<<<\\u30a9", {"\\u30a9", "\\u30a7"}, 2 }, { "&[before 3]\\u30a7<<<\\u30a9" "&\\u30A7=\\u30A7=\\u3047=\\uff6a" "&\\u30A8=\\u30A8=\\u3048=\\uff74", {"\\u30a9", "\\u30a7"}, 2 }, }; uint32_t i; for(i = 0; i<(sizeof(tests)/sizeof(tests[0])); i++) { genericRulesStarter(tests[i].rules, tests[i].data, tests[i].len); } #if 0 const char* rule1 = "&\\u30A7=\\u30A7=\\u3047=\\uff6a" "&\\u30A8=\\u30A8=\\u3048=\\uff74" "&[before 3]\\u30a7<<<\\u30c6|\\u30fc"; const char* rule2 = "&[before 3]\\u30a7<<<\\u30c6|\\u30fc" "&\\u30A7=\\u30A7=\\u3047=\\uff6a" "&\\u30A8=\\u30A8=\\u3048=\\uff74"; const char* test[] = { "\\u30c6\\u30fc\\u30bf", "\\u30c6\\u30a7\\u30bf", }; genericRulesStarter(rule1, test, sizeof(test)/sizeof(test[0])); genericRulesStarter(rule2, test, sizeof(test)/sizeof(test[0])); /* this piece of code should be in some sort of verbose mode */ /* it gets the collation elements for elements and prints them */ /* This is useful when trying to see whether the problem is */ { UErrorCode status = U_ZERO_ERROR; uint32_t i = 0; UCollationElements *it = NULL; uint32_t CE; UChar string[256]; uint32_t uStringLen; UCollator *coll = NULL; uStringLen = u_unescape(rule1, string, 256); coll = ucol_openRules(string, uStringLen, UCOL_DEFAULT, UCOL_DEFAULT, NULL, &status); /*coll = ucol_open("ja_JP_JIS", &status);*/ it = ucol_openElements(coll, string, 0, &status); for(i = 0; i < sizeof(test)/sizeof(test[0]); i++) { log_verbose("%s\n", test[i]); uStringLen = u_unescape(test[i], string, 256); ucol_setText(it, string, uStringLen, &status); while((CE=ucol_next(it, &status)) != UCOL_NULLORDER) { log_verbose("%08X\n", CE); } log_verbose("\n"); } ucol_closeElements(it); ucol_close(coll); } #endif } static void TestPrefixCompose(void) { const char* rule1 = "&\\u30a7<<<\\u30ab|\\u30fc=\\u30ac|\\u30fc"; /* const char* test[] = { "\\u30c6\\u30fc\\u30bf", "\\u30c6\\u30a7\\u30bf", }; */ { UErrorCode status = U_ZERO_ERROR; /*uint32_t i = 0;*/ /*UCollationElements *it = NULL;*/ /* uint32_t CE;*/ UChar string[256]; uint32_t uStringLen; UCollator *coll = NULL; uStringLen = u_unescape(rule1, string, 256); coll = ucol_openRules(string, uStringLen, UCOL_DEFAULT, UCOL_DEFAULT, NULL, &status); ucol_close(coll); } } /* [last variable] last variable value [last primary ignorable] largest CE for primary ignorable [last secondary ignorable] largest CE for secondary ignorable [last tertiary ignorable] largest CE for tertiary ignorable [top] guaranteed to be above all implicit CEs, for now and in the future (in 1.8) */ static void TestRuleOptions(void) { /* values here are hardcoded and are correct for the current UCA * when the UCA changes, one might be forced to change these * values. (\\u02d0, \\U00010FFFC etc...) */ static struct { const char *rules; const char *data[50]; const uint32_t len; } tests[] = { /* - all befores here amount to zero */ { "&[before 3][first tertiary ignorable]<< s2 weiv will check - could be a problem with utf-8 iterator cycheng@ca.ibm.c... but if we use the utf-8 iterator, i.e. s1 = efbfbc62 and s2 = eda08021, we have s1 < s2 weiv hmmm cycheng@ca.ibm.c... note that we have a standalone high surrogate weiv that doesn't sound right cycheng@ca.ibm.c... we got the same inconsistent results on AIX and Win2000 weiv so you have two strings, you convert them to utf-8 and to utf-16BE cycheng@ca.ibm.c... yes weiv and then do the comparison cycheng@ca.ibm.c... in one case, the input strings are in utf8, and in the other case the input strings are in utf-16be weiv utf-16 strings look like a little endian ones in the example you sent me weiv It could be a bug - let me try to test it out cycheng@ca.ibm.c... ok cycheng@ca.ibm.c... we can wait till the conf. call cycheng@ca.ibm.c... next weke weiv that would be great weiv hmmm weiv I might be wrong weiv let me play with it some more cycheng@ca.ibm.c... ok cycheng@ca.ibm.c... also please check s3 = 0x0e3a0062 and s4 = 0x0e400021. both are in utf-16be cycheng@ca.ibm.c... seems with icu 2.2 we have s3 > s4, but not in icu 2.4 that's built for db2 cycheng@ca.ibm.c... also s1 & s2 that I sent you earlier are also in utf-16be weiv ok cycheng@ca.ibm.c... i ask sherman to send you more inconsistent data weiv thanks cycheng@ca.ibm.c... the 4 strings we sent are just samples */ #if 0 static void Alexis(void) { UErrorCode status = U_ZERO_ERROR; UCollator *coll = ucol_open("", &status); const char utf16be[2][4] = { { (char)0xd8, (char)0x00, (char)0x00, (char)0x21 }, { (char)0xff, (char)0xfc, (char)0x00, (char)0x62 } }; const char utf8[2][4] = { { (char)0xed, (char)0xa0, (char)0x80, (char)0x21 }, { (char)0xef, (char)0xbf, (char)0xbc, (char)0x62 }, }; UCharIterator iterU161, iterU162; UCharIterator iterU81, iterU82; UCollationResult resU16, resU8; uiter_setUTF16BE(&iterU161, utf16be[0], 4); uiter_setUTF16BE(&iterU162, utf16be[1], 4); uiter_setUTF8(&iterU81, utf8[0], 4); uiter_setUTF8(&iterU82, utf8[1], 4); ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); resU16 = ucol_strcollIter(coll, &iterU161, &iterU162, &status); resU8 = ucol_strcollIter(coll, &iterU81, &iterU82, &status); if(resU16 != resU8) { log_err("different results\n"); } ucol_close(coll); } #endif #define CMSCOLL_ALEXIS2_BUFFER_SIZE 256 static void Alexis2(void) { UErrorCode status = U_ZERO_ERROR; UChar U16Source[CMSCOLL_ALEXIS2_BUFFER_SIZE], U16Target[CMSCOLL_ALEXIS2_BUFFER_SIZE]; char U16BESource[CMSCOLL_ALEXIS2_BUFFER_SIZE], U16BETarget[CMSCOLL_ALEXIS2_BUFFER_SIZE]; char U8Source[CMSCOLL_ALEXIS2_BUFFER_SIZE], U8Target[CMSCOLL_ALEXIS2_BUFFER_SIZE]; int32_t U16LenS = 0, U16LenT = 0, U16BELenS = 0, U16BELenT = 0, U8LenS = 0, U8LenT = 0; UConverter *conv = NULL; UCharIterator U16BEItS, U16BEItT; UCharIterator U8ItS, U8ItT; UCollationResult resU16, resU16BE, resU8; const char* pairs[][2] = { { "\\ud800\\u0021", "\\uFFFC\\u0062"}, { "\\u0435\\u0308\\u0334", "\\u0415\\u0334\\u0340" }, { "\\u0E40\\u0021", "\\u00A1\\u0021"}, { "\\u0E40\\u0021", "\\uFE57\\u0062"}, { "\\u5F20", "\\u5F20\\u4E00\\u8E3F"}, { "\\u0000\\u0020", "\\u0000\\u0020\\u0000"}, { "\\u0020", "\\u0020\\u0000"} /* 5F20 (my result here) 5F204E008E3F 5F20 (your result here) */ }; int32_t i = 0; UCollator *coll = ucol_open("", &status); if(status == U_FILE_ACCESS_ERROR) { log_data_err("Is your data around?\n"); return; } else if(U_FAILURE(status)) { log_err("Error opening collator\n"); return; } ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); conv = ucnv_open("UTF16BE", &status); for(i = 0; i < sizeof(pairs)/sizeof(pairs[0]); i++) { U16LenS = u_unescape(pairs[i][0], U16Source, CMSCOLL_ALEXIS2_BUFFER_SIZE); U16LenT = u_unescape(pairs[i][1], U16Target, CMSCOLL_ALEXIS2_BUFFER_SIZE); resU16 = ucol_strcoll(coll, U16Source, U16LenS, U16Target, U16LenT); log_verbose("Result of strcoll is %i\n", resU16); U16BELenS = ucnv_fromUChars(conv, U16BESource, CMSCOLL_ALEXIS2_BUFFER_SIZE, U16Source, U16LenS, &status); U16BELenT = ucnv_fromUChars(conv, U16BETarget, CMSCOLL_ALEXIS2_BUFFER_SIZE, U16Target, U16LenT, &status); /* use the original sizes, as the result from converter is in bytes */ uiter_setUTF16BE(&U16BEItS, U16BESource, U16LenS); uiter_setUTF16BE(&U16BEItT, U16BETarget, U16LenT); resU16BE = ucol_strcollIter(coll, &U16BEItS, &U16BEItT, &status); log_verbose("Result of U16BE is %i\n", resU16BE); if(resU16 != resU16BE) { log_verbose("Different results between UTF16 and UTF16BE for %s & %s\n", pairs[i][0], pairs[i][1]); } u_strToUTF8(U8Source, CMSCOLL_ALEXIS2_BUFFER_SIZE, &U8LenS, U16Source, U16LenS, &status); u_strToUTF8(U8Target, CMSCOLL_ALEXIS2_BUFFER_SIZE, &U8LenT, U16Target, U16LenT, &status); uiter_setUTF8(&U8ItS, U8Source, U8LenS); uiter_setUTF8(&U8ItT, U8Target, U8LenT); resU8 = ucol_strcollIter(coll, &U8ItS, &U8ItT, &status); if(resU16 != resU8) { log_verbose("Different results between UTF16 and UTF8 for %s & %s\n", pairs[i][0], pairs[i][1]); } } ucol_close(coll); ucnv_close(conv); } static void TestHebrewUCA(void) { UErrorCode status = U_ZERO_ERROR; const char *first[] = { "d790d6b8d79cd795d6bcd7a9", "d790d79cd79ed7a7d799d799d7a1", "d790d6b4d79ed795d6bcd7a9", }; char utf8String[3][256]; UChar utf16String[3][256]; int32_t i = 0, j = 0; int32_t sizeUTF8[3]; int32_t sizeUTF16[3]; UCollator *coll = ucol_open("", &status); /*ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);*/ for(i = 0; i < sizeof(first)/sizeof(first[0]); i++) { sizeUTF8[i] = u_parseUTF8(first[i], -1, utf8String[i], 256, &status); u_strFromUTF8(utf16String[i], 256, &sizeUTF16[i], utf8String[i], sizeUTF8[i], &status); log_verbose("%i: "); for(j = 0; j < sizeUTF16[i]; j++) { /*log_verbose("\\u%04X", utf16String[i][j]);*/ log_verbose("%04X", utf16String[i][j]); } log_verbose("\n"); } for(i = 0; i < sizeof(first)/sizeof(first[0])-1; i++) { for(j = i + 1; j < sizeof(first)/sizeof(first[0]); j++) { doTest(coll, utf16String[i], utf16String[j], UCOL_LESS); } } ucol_close(coll); } static void TestPartialSortKeyTermination(void) { const char* cases[] = { "\\u1234\\u1234\\udc00", "\\udc00\\ud800\\ud800" }; int32_t i = sizeof(UCollator); UErrorCode status = U_ZERO_ERROR; UCollator *coll = ucol_open("", &status); UCharIterator iter; UChar currCase[256]; int32_t length = 0; int32_t pKeyLen = 0; uint8_t key[256]; for(i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) { uint32_t state[2] = {0, 0}; length = u_unescape(cases[i], currCase, 256); uiter_setString(&iter, currCase, length); pKeyLen = ucol_nextSortKeyPart(coll, &iter, state, key, 256, &status); log_verbose("Done\n"); } ucol_close(coll); } static void TestSettings(void) { const char* cases[] = { "apple", "Apple" }; const char* locales[] = { "", "en" }; UErrorCode status = U_ZERO_ERROR; int32_t i = 0, j = 0; UChar source[256], target[256]; int32_t sLen = 0, tLen = 0; UCollator *collateObject = NULL; for(i = 0; i < sizeof(locales)/sizeof(locales[0]); i++) { collateObject = ucol_open(locales[i], &status); ucol_setStrength(collateObject, UCOL_PRIMARY); ucol_setAttribute(collateObject, UCOL_CASE_LEVEL , UCOL_OFF, &status); for(j = 1; j < sizeof(cases)/sizeof(cases[0]); j++) { sLen = u_unescape(cases[j-1], source, 256); source[sLen] = 0; tLen = u_unescape(cases[j], target, 256); source[tLen] = 0; doTest(collateObject, source, target, UCOL_EQUAL); } ucol_close(collateObject); } } static int32_t TestEqualsForCollator(const char* locName, UCollator *source, UCollator *target) { UErrorCode status = U_ZERO_ERROR; int32_t errorNo = 0; /*const UChar *sourceRules = NULL;*/ /*int32_t sourceRulesLen = 0;*/ UColAttributeValue french = UCOL_OFF; int32_t cloneSize = 0; if(!ucol_equals(source, target)) { log_err("Same collators, different address not equal\n"); errorNo++; } ucol_close(target); if(uprv_strcmp(ucol_getLocale(source, ULOC_REQUESTED_LOCALE, &status), ucol_getLocale(source, ULOC_ACTUAL_LOCALE, &status)) == 0) { /* currently, safeClone is implemented through getRules/openRules * so it is the same as the test below - I will comment that test out. */ /* real thing */ target = ucol_safeClone(source, NULL, &cloneSize, &status); if(U_FAILURE(status)) { log_err("Error creating clone\n"); errorNo++; return errorNo; } if(!ucol_equals(source, target)) { log_err("Collator different from it's clone\n"); errorNo++; } french = ucol_getAttribute(source, UCOL_FRENCH_COLLATION, &status); if(french == UCOL_ON) { ucol_setAttribute(target, UCOL_FRENCH_COLLATION, UCOL_OFF, &status); } else { ucol_setAttribute(target, UCOL_FRENCH_COLLATION, UCOL_ON, &status); } if(U_FAILURE(status)) { log_err("Error setting attributes\n"); errorNo++; return errorNo; } if(ucol_equals(source, target)) { log_err("Collators same even when options changed\n"); errorNo++; } ucol_close(target); /* commented out since safeClone uses exactly the same technique */ /* sourceRules = ucol_getRules(source, &sourceRulesLen); target = ucol_openRules(sourceRules, sourceRulesLen, UCOL_DEFAULT, UCOL_DEFAULT, &parseError, &status); if(U_FAILURE(status)) { log_err("Error instantiating target from rules\n"); errorNo++; return errorNo; } if(!ucol_equals(source, target)) { log_err("Collator different from collator that was created from the same rules\n"); errorNo++; } ucol_close(target); */ } return errorNo; } static void TestEquals(void) { /* ucol_equals is not currently a public API. There is a chance that it will become * something like this, but currently it is only used by RuleBasedCollator::operator== */ /* test whether the two collators instantiated from the same locale are equal */ UErrorCode status = U_ZERO_ERROR; UParseError parseError; int32_t noOfLoc = uloc_countAvailable(); const char *locName = NULL; UCollator *source = NULL, *target = NULL; int32_t i = 0; const char* rules[] = { "&l < lj <<< Lj <<< LJ", "&n < nj <<< Nj <<< NJ", "&ae <<< \\u00e4", "&AE <<< \\u00c4" }; /* const char* badRules[] = { "&l <<< Lj", "&n < nj <<< nJ <<< NJ", "&a <<< \\u00e4", "&AE <<< \\u00c4 <<< x" }; */ UChar sourceRules[1024], targetRules[1024]; int32_t sourceRulesSize = 0, targetRulesSize = 0; int32_t rulesSize = sizeof(rules)/sizeof(rules[0]); for(i = 0; i < rulesSize; i++) { sourceRulesSize += u_unescape(rules[i], sourceRules+sourceRulesSize, 1024 - sourceRulesSize); targetRulesSize += u_unescape(rules[rulesSize-i-1], targetRules+targetRulesSize, 1024 - targetRulesSize); } source = ucol_openRules(sourceRules, sourceRulesSize, UCOL_DEFAULT, UCOL_DEFAULT, &parseError, &status); if(status == U_FILE_ACCESS_ERROR) { log_data_err("Is your data around?\n"); return; } else if(U_FAILURE(status)) { log_err("Error opening collator\n"); return; } target = ucol_openRules(targetRules, targetRulesSize, UCOL_DEFAULT, UCOL_DEFAULT, &parseError, &status); if(!ucol_equals(source, target)) { log_err("Equivalent collators not equal!\n"); } ucol_close(source); ucol_close(target); source = ucol_open("root", &status); target = ucol_open("root", &status); log_verbose("Testing root\n"); if(!ucol_equals(source, source)) { log_err("Same collator not equal\n"); } if(TestEqualsForCollator(locName, source, target)) { log_err("Errors for root\n", locName); } ucol_close(source); for(i = 0; i= 0 && i <= TST_UCOL_MAX_INPUT) { log_verbose("%08X\t%08X\n", i, uprv_uca_getImplicitFromRaw(i)); } } static void TestImplicitGeneration(void) { UErrorCode status = U_ZERO_ERROR; UChar32 last = 0; UChar32 current; UChar32 i = 0, j = 0; UChar32 roundtrip = 0; UChar32 lastBottom = 0; UChar32 currentBottom = 0; UChar32 lastTop = 0; UChar32 currentTop = 0; UCollator *coll = ucol_open("root", &status); if(U_FAILURE(status)) { log_err("Couldn't open UCA\n"); return; } uprv_uca_getRawFromImplicit(0xE20303E7); for (i = 0; i <= TST_UCOL_MAX_INPUT; ++i) { current = uprv_uca_getImplicitFromRaw(i) & fourBytes; /* check that it round-trips AND that all intervening ones are illegal*/ roundtrip = uprv_uca_getRawFromImplicit(current); if (roundtrip != i) { log_err("No roundtrip %08X\n", i); } if (last != 0) { for (j = last + 1; j < current; ++j) { roundtrip = uprv_uca_getRawFromImplicit(j); /* raise an error if it *doesn't* find an error*/ if (roundtrip != -1) { log_err("Fails to recognize illegal %08X\n", j); } } } /* now do other consistency checks*/ lastBottom = last & bottomByte; currentBottom = current & bottomByte; lastTop = last & topByte; currentTop = current & topByte; /* print out some values for spot-checking*/ if (lastTop != currentTop || i == 0x10000 || i == 0x110000) { showImplicit(i-3); showImplicit(i-2); showImplicit(i-1); showImplicit(i); showImplicit(i+1); showImplicit(i+2); } last = current; if(uprv_uca_getCodePointFromRaw(uprv_uca_getRawFromCodePoint(i)) != i) { log_err("No raw <-> code point roundtrip for 0x%08X\n", i); } } showImplicit(TST_UCOL_MAX_INPUT-2); showImplicit(TST_UCOL_MAX_INPUT-1); showImplicit(TST_UCOL_MAX_INPUT); ucol_close(coll); } /** * Iterate through the given iterator, checking to see that all the strings * in the expected array are present. * @param expected array of strings we expect to see, or NULL * @param expectedCount number of elements of expected, or 0 */ static int32_t checkUEnumeration(const char* msg, UEnumeration* iter, const char** expected, int32_t expectedCount) { UErrorCode ec = U_ZERO_ERROR; int32_t i = 0, n, j, bit; int32_t seenMask = 0; U_ASSERT(expectedCount >= 0 && expectedCount < 31); /* [sic] 31 not 32 */ n = uenum_count(iter, &ec); if (!assertSuccess("count", &ec)) return -1; log_verbose("%s = [", msg); for (;; ++i) { const char* s = uenum_next(iter, NULL, &ec); if (!assertSuccess("snext", &ec) || s == NULL) break; if (i != 0) log_verbose(","); log_verbose("%s", s); /* check expected list */ for (j=0, bit=1; j