From a8e4cbac2893edddc2fca23c5b39ce4a7796ebde Mon Sep 17 00:00:00 2001 From: Klaus Date: Tue, 10 Nov 2026 10:23:19 +0900 Subject: [PATCH] =?UTF-8?q?refactor(i18n):=20=EC=BD=94=EB=93=9C=ED=8F=AC?= =?UTF-8?q?=EC=9D=B8=ED=8A=B8=20=EA=B8=B0=EB=B0=98=20=EB=AC=B8=EC=9E=90?= =?UTF-8?q?=EC=97=B4=20=EB=B6=84=ED=95=A0=EC=9D=84=20=EA=B3=B5=ED=86=B5?= =?UTF-8?q?=ED=99=94=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../i18n/translation/CodePointTextSplitter.kt | 41 +++++ .../translation/PapagoTranslationService.kt | 12 +- .../translation/CodePointTextSplitterTest.kt | 157 ++++++++++++++++++ .../PapagoTranslationServiceTest.kt | 9 + 4 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitter.kt create mode 100644 src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitterTest.kt diff --git a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitter.kt b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitter.kt new file mode 100644 index 00000000..5312b954 --- /dev/null +++ b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitter.kt @@ -0,0 +1,41 @@ +package kr.co.vividnext.sodalive.i18n.translation + +internal fun splitTextByCodePoint(text: String, maxSize: Int, codePointSize: (Int) -> Int): List { + val chunks = mutableListOf() + var start = 0 + do { + val end = findChunkEnd(text, start, maxSize, codePointSize) + chunks.add(text.substring(start, end)) + start = end + } while (start < text.length) + return chunks +} + +internal fun takeTextPrefixByCodePoint(text: String, maxSize: Int, codePointSize: (Int) -> Int): String { + return text.substring(0, findChunkEnd(text, 0, maxSize, codePointSize)) +} + +internal fun utf8CodePointSize(codePoint: Int): Int { + return when { + codePoint <= 0x7F -> 1 + codePoint <= 0x7FF -> 2 + codePoint in 0xD800..0xDFFF -> 1 + codePoint <= 0xFFFF -> 3 + else -> 4 + } +} + +private fun findChunkEnd(text: String, start: Int, maxSize: Int, codePointSize: (Int) -> Int): Int { + var end = start + var size = 0 + while (end < text.length) { + val codePoint = text.codePointAt(end) + val nextSize = codePointSize(codePoint) + if (size + nextSize > maxSize) { + break + } + size += nextSize + end += Character.charCount(codePoint) + } + return end +} diff --git a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationService.kt b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationService.kt index 24f79539..71ec3aaa 100644 --- a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationService.kt +++ b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationService.kt @@ -78,17 +78,7 @@ class PapagoTranslationService( } private fun splitText(text: String): List { - val chunks = mutableListOf() - var start = 0 - do { - var end = minOf(start + MAX_REQUEST_TEXT_LENGTH, text.length) - if (end < text.length && text[end - 1].isHighSurrogate() && text[end].isLowSurrogate()) { - end -= 1 - } - chunks.add(text.substring(start, end)) - start = end - } while (start < text.length) - return chunks + return splitTextByCodePoint(text, MAX_REQUEST_TEXT_LENGTH, Character::charCount) } private fun validateLanguages(sourceLanguage: String, targetLanguage: String): Boolean { diff --git a/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitterTest.kt b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitterTest.kt new file mode 100644 index 00000000..61ec5c1d --- /dev/null +++ b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/CodePointTextSplitterTest.kt @@ -0,0 +1,157 @@ +package kr.co.vividnext.sodalive.i18n.translation + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource + +class CodePointTextSplitterTest { + @Test + @DisplayName("정확히 5000 UTF-16 단위인 한글과 이모지는 한 조각으로 유지한다") + fun shouldKeepTextAtUtf16LimitInOneChunk() { + val source = "가".repeat(4998) + "\uD83D\uDE00" + + assertEquals(listOf(source), splitTextByCodePoint(source, 5000, Character::charCount)) + } + + @Test + @DisplayName("5001 UTF-16 단위 원문은 5000 단위와 나머지로 분할한다") + fun shouldSplitTextAboveUtf16Limit() { + val first = "a".repeat(5000) + + assertEquals(listOf(first, "b"), splitTextByCodePoint(first + "b", 5000, Character::charCount)) + } + + @Test + @DisplayName("UTF-16 경계에 걸친 서로게이트 쌍을 다음 조각에 함께 둔다") + fun shouldKeepSurrogatePairTogetherAtUtf16Boundary() { + val first = "a".repeat(4999) + val second = "\uD83D\uDE00b" + + assertEquals(listOf(first, second), splitTextByCodePoint(first + second, 5000, Character::charCount)) + } + + @Test + @DisplayName("정확히 10000 UTF-8 바이트인 한글과 이모지는 한 조각으로 유지한다") + fun shouldKeepTextAtUtf8LimitInOneChunk() { + val source = "가".repeat(3332) + "\uD83D\uDE00" + + assertEquals(10000, source.toByteArray(Charsets.UTF_8).size) + assertEquals(listOf(source), splitTextByCodePoint(source, 10000, ::utf8CodePointSize)) + } + + @Test + @DisplayName("10001 UTF-8 바이트 원문은 10000 바이트와 나머지로 분할한다") + fun shouldSplitTextAboveUtf8Limit() { + val first = "a".repeat(10000) + + assertEquals(listOf(first, "b"), splitTextByCodePoint(first + "b", 10000, ::utf8CodePointSize)) + } + + @ParameterizedTest + @ValueSource(strings = ["가", "\uD83D\uDE00"]) + @DisplayName("UTF-8 경계의 한글과 이모지를 자르거나 한도를 초과하지 않는다") + fun shouldKeepWholeCodePointAtUtf8Boundary(boundaryText: String) { + val first = "a".repeat(9998) + val second = boundaryText + "b" + + assertEquals(listOf(first, second), splitTextByCodePoint(first + second, 10000, ::utf8CodePointSize)) + } + + @Test + @DisplayName("UTF-16 분할 후 한글과 이모지 및 모든 공백을 원문 그대로 재구성한다") + fun shouldReconstructOriginalTextAfterUtf16Splitting() { + val source = " \t가\uD83D\uDE00\r\n".repeat(3000) + " " + + val chunks = splitTextByCodePoint(source, 5000, Character::charCount) + + assertEquals(source, chunks.joinToString("")) + assertTrue(chunks.all { it.length in 1..5000 }) + chunks.forEach { assertEquals(it, String(it.toByteArray(Charsets.UTF_8), Charsets.UTF_8)) } + } + + @Test + @DisplayName("UTF-8 분할 후 한글과 이모지 및 모든 공백을 원문 그대로 재구성한다") + fun shouldReconstructOriginalTextAfterUtf8Splitting() { + val source = " \t가\uD83D\uDE00\r\n".repeat(3000) + " " + + val chunks = splitTextByCodePoint(source, 10000, ::utf8CodePointSize) + + assertEquals(source, chunks.joinToString("")) + assertTrue(chunks.all { it.toByteArray(Charsets.UTF_8).size in 1..10000 }) + chunks.forEach { assertEquals(it, String(it.toByteArray(Charsets.UTF_8), Charsets.UTF_8)) } + } + + @Test + @DisplayName("빈 raw 입력은 빈 문자열 하나를 가진 조각 목록을 반환한다") + fun shouldPreserveEmptyRawInputAsOneChunk() { + assertEquals(listOf(""), splitTextByCodePoint("", 5000, Character::charCount)) + assertEquals(listOf(""), splitTextByCodePoint("", 10000, ::utf8CodePointSize)) + } + + @Test + @DisplayName("감지 prefix는 정확히 100000 UTF-8 바이트를 보존하고 tail을 제외한다") + fun shouldTakeExactUtf8PrefixWithoutTail() { + val prefix = "가".repeat(33332) + "\uD83D\uDE00" + val source = prefix + "excluded-tail" + + val result = takeTextPrefixByCodePoint(source, 100000, ::utf8CodePointSize) + + assertEquals(prefix, result) + assertEquals(100000, result.toByteArray(Charsets.UTF_8).size) + } + + @ParameterizedTest + @ValueSource(strings = ["가", "\uD83D\uDE00"]) + @DisplayName("감지 prefix 경계의 코드포인트가 넘치면 남은 바이트를 채우지 않고 멈춘다") + fun shouldStopPrefixBeforeOverflowingCodePoint(boundaryText: String) { + val prefix = "a".repeat(99999) + + assertEquals(prefix, takeTextPrefixByCodePoint(prefix + boundaryText + "tail", 100000, ::utf8CodePointSize)) + } + + @ParameterizedTest + @ValueSource(strings = ["", " \t\n", "한글\uD83D\uDE00"]) + @DisplayName("한도보다 짧은 prefix는 빈 문자열과 공백도 변경하지 않는다") + fun shouldPreserveTextWithinPrefixLimit(source: String) { + assertEquals(source, takeTextPrefixByCodePoint(source, 100000, ::utf8CodePointSize)) + } + + @Test + @DisplayName("prefix는 첫 조각의 경계만 검사하고 나머지 원문을 순회하지 않는다") + fun shouldStopScanningAfterFirstPrefixBoundary() { + val prefix = "a".repeat(100000) + val source = prefix + "가" + "tail".repeat(25000) + var measuredCodePoints = 0 + + val result = takeTextPrefixByCodePoint(source, 100000) { codePoint -> + measuredCodePoints += 1 + utf8CodePointSize(codePoint) + } + + assertEquals(prefix, result) + assertEquals(100001, measuredCodePoints) + } + + @ParameterizedTest + @ValueSource( + ints = [0, 0x7F, 0x80, 0x7FF, 0x800, 0xD7FF, 0xD800, 0xDBFF, 0xDC00, 0xDFFF, 0xE000, 0xFFFF, 0x10000, 0x1F600, 0x10FFFF] + ) + @DisplayName("UTF-8 코드포인트 크기는 JVM 인코딩의 바이트 수와 같다") + fun shouldMatchUtf8EncodedCodePointSize(codePoint: Int) { + val text = String(Character.toChars(codePoint)) + + assertEquals(text.toByteArray(Charsets.UTF_8).size, utf8CodePointSize(codePoint)) + } + + @Test + @DisplayName("짝 없는 surrogate도 JVM UTF-8 인코딩 크기로 계산하며 원문을 보존한다") + fun shouldPreserveUnpairedSurrogateWithItsEncodedSize() { + val first = "a".repeat(9999) + "\uD800" + + assertEquals(10000, first.toByteArray(Charsets.UTF_8).size) + assertEquals(listOf(first, "b"), splitTextByCodePoint(first + "b", 10000, ::utf8CodePointSize)) + } +} diff --git a/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationServiceTest.kt b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationServiceTest.kt index f75db2c6..35e0ad76 100644 --- a/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationServiceTest.kt +++ b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/PapagoTranslationServiceTest.kt @@ -61,6 +61,15 @@ class PapagoTranslationServiceTest @Autowired constructor( assertEquals(listOf("translated"), translate(source).translatedText) } + @ParameterizedTest + @ValueSource(strings = ["", " \t\n"]) + @DisplayName("빈 raw 문자열과 공백 원문도 변경 없이 한 번 요청한다") + fun shouldPreserveEmptyAndWhitespaceRawText(source: String) { + expectTranslation(source, "translated") + + assertEquals(listOf("translated"), translate(source).translatedText) + } + @Test @DisplayName("5,001자 원문은 두 번 요청하고 번역을 구분자 없이 결합한다") fun shouldSplit5001UnitsAndJoinTranslationsWithoutSeparator() {