We localise KOPI to many different languages, including traditional and simplified Chinese. As I was working through the Christmas edition, I happily typed in “Christmas” in the traditional script (聖誕節), saved everything, compiled the localised strings, and assumed that the simplified version (圣诞节) will also work. To my surprise, I got 圣∎节. (Well, not exactly end-of-proof symbol, but the horrible glyph missing question mark.) I certainly didn’t feel like going through all the strings in our game to eyeball everything, so I had to throw together a simple automation test to verify that all messages in our game are representable in our fonts.
The trap is mainly in handling of Unicode characters. The usual ABC can easily fit in one byte, but once you get past that, depending on platform, TCHAR (aka wchar_t) will also change. on x86_64 MSVC Windows, sizeof(wchar_t) is 2; on x86_64 clang Linux, it’s 4; on arm64 Mac, it’s also 4. The more unusual the characters get, the more likely you’re to need multi-byte encoding. For example, 鬣蜥海峡 is U+9B23, U+8725, U+6D77, U+5CE1; in UTF-8, it will be encoded as 0xe9 0xac 0xa3 0xe8 0x9c 0xa5 0xe6 0xb5 0xb7 0xe5 0xb3 0xa1. In other words, iterating TCHAR by TCHAR in an FString checking using FCharacterList::GetCharacter is a trap.
FSlateFontCache& FontCache = FSlateApplication::Get().GetRenderer()->GetFontCache();
FCharacterList& CharList = FontCache.GetCharacterList(...);
for (const TCHAR C : String) {
const FCharacterEntry& Entry = CharList.GetCharacter(Character, ...);
}Instead, we must use FShapedTextCache.
auto Contains = [this](const UFont *Font, const FString &S) {
FSlateFontInfo FontInfo;
FontInfo.FontObject = Font;
FontInfo.Size = 20;
const FCachedShapedTextKey Key(FTextRange(0, S.Len()),
1.f,
FShapedTextContext(ETextShapingMethod::Auto,
TextBiDi::ETextDirection::LeftToRight),
FontInfo);
const TSharedRef<FSlateFontServices> SlateFontServices =
FSlateApplication::Get().GetRenderer()->GetFontServices();
const auto FontCache = SlateFontServices->GetFontCache();
const auto ShapedCache = FShapedTextCache::Create(FontCache);
const auto Sequence = ShapedCache->FindOrAddShapedText(Key, *S);
for (const FShapedGlyphEntry &GlyphEntry : Sequence->GetGlyphsToRender()) {
if (!GlyphEntry.HasValidGlyph()) {
return false;
}
}
return true;
};The whole beast is for your testing pleasure here.
BEGIN_DEFINE_SPEC(FKopiGlyphPresenceTest, "...Localisation.GlyphPresence",
EAutomationTestFlags::EditorContext | AutomationTestFlags::ProductFilter)
END_DEFINE_SPEC(FKopiGlyphPresenceTest)
TArray<FString> GetStrings(const FString &LocResDir) {
TArray<FString> Out;
FTextLocalizationResource Resource;
constexpr int32 Priority = 0;
Resource.LoadFromDirectory(LocResDir, Priority);
for (const auto &Pair : Resource.Entries) {
FString Item = *Pair.Value.LocalizedString;
Item.ReplaceCharInline('\n', '.');
Item.ReplaceCharInline('\r', '.');
Out.Add(Item);
}
return Out;
}
void FKopiGlyphPresenceTest::Define() {
It("All fonts display all text", [&] {
auto Contains = [this](const UFont *Font, const FString &S) {
FSlateFontInfo FontInfo;
FontInfo.FontObject = Font;
FontInfo.Size = 20;
const FCachedShapedTextKey Key(FTextRange(0, S.Len()),
1.f,
FShapedTextContext(ETextShapingMethod::Auto,
TextBiDi::ETextDirection::LeftToRight),
FontInfo);
const TSharedRef<FSlateFontServices> SlateFontServices =
FSlateApplication::Get().GetRenderer()->GetFontServices();
const TSharedRef<FSlateFontCache> FontCache =
SlateFontServices->GetFontCache();
const TSharedRef<FShapedTextCache> ShapedCache
FShapedTextCache::Create(FontCache);
const FShapedGlyphSequenceRef Sequence =
ShapedCache->FindOrAddShapedText(Key, *S);
for (const FShapedGlyphEntry &GlyphEntry : Sequence->GetGlyphsToRender()) {
if (!GlyphEntry.HasValidGlyph()) {
return false;
}
}
return true;
};
// for all our "text" fonts...
const UFont *Headline = LoadObject<UFont>(nullptr, TEXT("/Game/UI/Fonts/..."));
const UFont *InGame = LoadObject<UFont>(nullptr, TEXT("/Game/UI/Fonts/..."));
const UFont *InUI = LoadObject<UFont>(nullptr, TEXT("/Game/UI/Fonts/..."));
// a quick positive test. ABC are definitely valid glyphs.
const FString Valid = UTF8TEXT("ABC");
TestTrue("Must be valid", Contains(Headline, Valid));
TestTrue("Must be valid", Contains(InGame, Valid));
TestTrue("Must be valid", Contains(InUI, Valid));
// a quick negative test. 鬣蜥海峡 is in China Mieville's 地疤, not in KOPI.
const FString Invalid = UTF8TEXT("鬣蜥海峡");
TestFalse("Must be invalid", Contains(Headline, Invalid));
TestFalse("Must be invalid", Contains(InGame, Invalid));
TestFalse("Must be invalid", Contains(InUI, Invalid));
// and now for all our locales...
for (const FString L : {"en", "cs", "zh-Hans", "zh-Hant", "es-ES", "ms-MY"}) {
FString Dir = FPaths::ProjectContentDir() / TEXT("Localization/Game/" + L);
const TArray<FString> Strings = GetStrings(Dir);
for (const FString &String : Strings) {
TestTrue("...", Contains(Headline, String));
TestTrue("...", Contains(InGame, String));
TestTrue("...", Contains(InUI, String));
}
}
});
}