klaus/sodalive-backend-spring-boot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

AWS 번역 공급자 추가 #469

Merged
klaus merged 12 commits from test into main 2026-11-10 05:25:25 +00:00
Conversation 0 Commits 12 Files Changed 50
+204 -32
1 changed files with 204 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 9d29355bf2 - Show all commits
src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/LanguageProviderCacheEndToEndTest.kt
+204 -32
View File
@@ -14,23 +14,40 @@ import kr.co.vividnext.sodalive.member.MemberRole
import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.adapter.out.persistence.CreatorCommunityTranslationRepository
import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.application.CreatorCommunityTranslationService
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNotEquals
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertSame
import org.junit.jupiter.api.Assertions.assertThrows
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.ArgumentCaptor
import org.mockito.Mockito
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
import org.springframework.boot.test.context.TestConfiguration
import org.springframework.boot.test.mock.mockito.SpyBean
import org.springframework.boot.test.system.CapturedOutput
import org.springframework.boot.test.system.OutputCaptureExtension
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Import
import org.springframework.transaction.PlatformTransactionManager
import org.springframework.transaction.annotation.Propagation
import org.springframework.transaction.annotation.Transactional
import org.springframework.transaction.support.TransactionTemplate
import software.amazon.awssdk.core.exception.SdkClientException
import software.amazon.awssdk.services.comprehend.ComprehendClient
import software.amazon.awssdk.services.comprehend.model.DetectDominantLanguageRequest
import software.amazon.awssdk.services.comprehend.model.DetectDominantLanguageResponse
import software.amazon.awssdk.services.comprehend.model.DominantLanguage
import software.amazon.awssdk.services.translate.TranslateClient
import software.amazon.awssdk.services.translate.model.TranslateTextRequest
import software.amazon.awssdk.services.translate.model.TranslateTextResponse
import java.time.LocalDateTime
import java.util.UUID
@@ -52,9 +69,12 @@ import java.util.UUID
TranslationJobScheduler::class,
ResourceTranslationJobScheduler::class,
CreatorCommunityTranslationService::class,
AwsLanguageDetectionProvider::class,
AwsTranslationService::class,
LanguageProviderCacheEndToEndTest.Providers::class
)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
@ExtendWith(OutputCaptureExtension::class)
class LanguageProviderCacheEndToEndTest @Autowired constructor(
private val detector: LanguageDetectionCacheService,
private val detectionRepository: LanguageDetectionResultRepository,
@@ -67,9 +87,11 @@ class LanguageProviderCacheEndToEndTest @Autowired constructor(
private val members: MemberRepository,
private val translations: CreatorCommunityTranslationRepository,
private val papagoDetection: RecordingDetectionProvider,
private val secondaryDetection: RecordingDetectionProvider,
private val papagoTranslation: RecordingTranslationProvider,
private val secondaryTranslation: RecordingTranslationProvider,
private val comprehendClient: ComprehendClient,
private val translateClient: TranslateClient,
private val awsTranslation: AwsTranslationService,
private val coordinator: TranslationCoordinator,
transactionManager: PlatformTransactionManager
) {
private val transaction = TransactionTemplate(transactionManager)
@@ -79,34 +101,70 @@ class LanguageProviderCacheEndToEndTest @Autowired constructor(
@BeforeEach
fun useH2CompatibleClaimQuery() {
Mockito.reset(comprehendClient, translateClient)
papagoDetection.queries.clear()
papagoTranslation.requests.clear()
transaction.executeWithoutResult {
translations.deleteAll()
jobRepository.deleteAll()
memoryRepository.deleteAll()
detectionRepository.deleteAll()
}
Mockito.doAnswer {
jobRepository.findAll().filter { it.status == TranslationJobStatus.PENDING }.mapNotNull { it.id }
}.`when`(jobRepository).findNextPendingGroupJobIdsForUpdate(Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now())
}
@Test
@DisplayName("보조 공급자의 감지를 H2에 귀속 저장하고 외부 호출 없이 캐시를 재사용한다")
@DisplayName("동일 AWS 감지 prefix의 서로 다른 전체 원문을 H2에 구분 저장하고 무호출 캐시를 재사용한다")
fun shouldPersistSecondaryDetectionAndReuseCache() {
val source = "provider source ${UUID.randomUUID()}"
assertEquals(DetectedLanguage("ko", AWS), detector.detectWithCache(source))
assertEquals(DetectedLanguage("ko", AWS), detector.detectWithCache(source))
val detection = transaction.execute {
detectionRepository.findBySourceHashAndProviderAndNormalizationVersion(
SourceTextNormalizer.hash(source),
"aws",
"v1"
)
}!!
assertEquals("aws", detection.provider)
assertEquals("ko", detection.detectedLanguage)
assertEquals(1, papagoDetection.queries.size)
assertEquals(1, secondaryDetection.queries.size)
val prefix = "가".repeat(33332) + "\uD83D\uDE00"
val sources = listOf(prefix + " first ${UUID.randomUUID()}", prefix + " second ${UUID.randomUUID()}")
Mockito.`when`(comprehendClient.detectDominantLanguage(Mockito.any(DetectDominantLanguageRequest::class.java)))
.thenReturn(detectionResponse("ko"))
sources.forEach { assertEquals(DetectedLanguage("ko", AWS), detector.detectWithCache(it)) }
sources.forEach { assertEquals(DetectedLanguage("ko", AWS), detector.detectWithCache(it)) }
val detections = transaction.execute { detectionRepository.findAll() }!!
assertEquals(100000, prefix.toByteArray(Charsets.UTF_8).size)
assertNotEquals(SourceTextNormalizer.hash(sources[0]), SourceTextNormalizer.hash(sources[1]))
assertEquals(2, detections.size)
assertEquals(2, detections.map { it.id }.distinct().size)
assertEquals(sources.map { SourceTextNormalizer.hash(it) }.toSet(), detections.map { it.sourceHash }.toSet())
detections.forEach {
assertEquals("aws", it.provider)
assertEquals("ko", it.detectedLanguage)
assertEquals("v1", it.normalizationVersion)
assertNotEquals(SourceTextNormalizer.hash(prefix), it.sourceHash)
}
assertEquals(sources, papagoDetection.queries)
assertDetectionRequests(prefix, prefix)
}
@Test
@DisplayName("보조 공급자의 번역을 H2에 귀속 저장하고 무호출 캐시와 실제 표시를 재사용한다")
@DisplayName("AWS 첫 감지 후보가 공백이면 뒤 후보를 쓰거나 H2에 캐시하지 않고 다음 요청도 다시 호출한다")
fun shouldNotCacheBlankFirstDetectionCandidate() {
val source = "uncached detection ${UUID.randomUUID()}"
Mockito.`when`(comprehendClient.detectDominantLanguage(Mockito.any(DetectDominantLanguageRequest::class.java)))
.thenReturn(detectionResponse(" ", "ko"))
assertNull(detector.detectWithCache(source))
assertNull(detector.detectWithCache(source))
assertEquals(0L, transaction.execute { detectionRepository.count() })
assertEquals(listOf(source, source), papagoDetection.queries)
assertDetectionRequests(source, source)
}
@Test
@DisplayName("실제 AWS 분할 번역을 H2에 전체 원문으로 저장하고 worker 완료 및 무호출 표시를 재사용한다")
fun shouldPersistSecondaryTranslationAndReuseCacheThroughWorkerAndMaterializer() {
val source = "translation source ${UUID.randomUUID()}"
val first = "가".repeat(3332) + "\uD83D\uDE00"
val tail = "나 tail ${UUID.randomUUID()}"
val source = first + tail
Mockito.`when`(translateClient.translateText(Mockito.any(TranslateTextRequest::class.java)))
.thenReturn(translationResponse("first "), translationResponse("tail"))
val postId = persistPost(source)
schedule(postId, source)
assertTrue(worker.processNextGroup())
@@ -120,14 +178,23 @@ class LanguageProviderCacheEndToEndTest @Autowired constructor(
)
}
assertNotNull(memory)
assertEquals("aws", memory!!.provider)
assertEquals("AWS-v1", memory.providerVersion)
assertEquals("AWS:0", memory.translatedText)
assertEquals(10000, first.toByteArray(Charsets.UTF_8).size)
assertTrue(source.toByteArray(Charsets.UTF_8).size > 10000)
assertEquals(1L, transaction.execute { memoryRepository.count() })
assertEquals(source, memory!!.sourceText)
assertEquals(SourceTextNormalizer.hash(source), memory.sourceHash)
assertEquals("ko", memory.sourceLanguage)
assertEquals("en", memory.targetLanguage)
assertEquals("aws", memory.provider)
assertEquals("v1", memory.normalizationVersion)
assertEquals("translate-2017-07-01", memory.providerVersion)
assertEquals("first tail", memory.translatedText)
assertEquals(
TranslationJobStatus.COMPLETED,
transaction.execute { jobRepository.findAll().single { it.resourceId == postId }.status }
)
assertEquals("AWS:0", displayService.findDisplayContents(listOf(postId), "en")[postId]?.content)
assertEquals("first tail", displayService.findDisplayContents(listOf(postId), "en")[postId]?.content)
assertEquals(true, displayService.findDisplayContents(listOf(postId), "en")[postId]?.hasValidTranslation)
transaction.executeWithoutResult {
translations.deleteAll()
@@ -137,9 +204,117 @@ class LanguageProviderCacheEndToEndTest @Autowired constructor(
schedule(postId, source)
assertTrue(worker.processNextGroup())
assertEquals(1, papagoTranslation.requests.size)
assertEquals(1, secondaryTranslation.requests.size)
assertEquals(listOf(source), secondaryTranslation.requests.single().texts)
assertEquals("AWS:0", displayService.findDisplayContents(listOf(postId), "en")[postId]?.content)
assertEquals(listOf(source), papagoTranslation.requests.single().texts)
assertTranslationRequests(first, tail)
assertEquals(1L, transaction.execute { memoryRepository.count() })
assertEquals(
TranslationJobStatus.COMPLETED,
transaction.execute { jobRepository.findAll().single { it.resourceId == postId }.status }
)
assertEquals("first tail", displayService.findDisplayContents(listOf(postId), "en")[postId]?.content)
assertEquals(true, displayService.findDisplayContents(listOf(postId), "en")[postId]?.hasValidTranslation)
}
@Test
@DisplayName("AWS 중간 조각 오류는 부분 결과와 저장을 버리고 worker에 원인 없는 안전한 오류만 남긴다")
fun shouldDiscardPartialAwsTranslationAndPersistSanitizedWorkerFailure(output: CapturedOutput) {
val first = "가".repeat(3332) + "\uD83D\uDE00"
val broken = "나".repeat(3333) + "b"
val source = first + broken + "synthetic-source-marker ${UUID.randomUUID()}"
val request = TranslateRequest(listOf(source), "ko", "en")
failTranslationAfter(first)
assertEquals(emptyList<String>(), awsTranslation.translate(request).translatedText)
assertTranslationRequests(first, broken)
val error = assertThrows(IllegalStateException::class.java) { coordinator.translate(request) }
assertEquals("all translation providers failed", error.message)
assertNull(error.cause)
assertTranslationRequests(first, broken, first, broken)
val postId = persistPost(source)
schedule(postId, source)
assertTrue(worker.processNextGroup())
val job = transaction.execute { jobRepository.findAll().single { it.resourceId == postId } }!!
assertEquals(TranslationJobStatus.PENDING, job.status)
assertEquals(1, job.retryCount)
assertEquals("all translation providers failed", job.lastErrorMessage)
assertEquals(0L, transaction.execute { memoryRepository.count() })
assertEquals(0L, transaction.execute { translations.count() })
assertFalse(materializer.materialize(LanguageTranslationTargetType.CREATOR_COMMUNITY, postId, "en"))
assertEquals(source, displayService.findDisplayContents(listOf(postId), "en")[postId]?.content)
assertEquals(false, displayService.findDisplayContents(listOf(postId), "en")[postId]?.hasValidTranslation)
assertEquals(listOf(request, request), papagoTranslation.requests)
assertTranslationRequests(first, broken, first, broken, first, broken)
assertFalse(output.all.contains("synthetic-sensitive-sdk-marker"))
assertFalse(output.all.contains("synthetic-source-marker"))
assertFalse(output.all.contains(source))
}
@Test
@DisplayName("테스트 전용 AWS 우선 순서에서 실제 부분 실패 뒤 Papago에 여러 원문 전체 요청을 그대로 넘긴다")
fun shouldFallbackEntireOriginalRequestAfterActualAwsPartialFailure(output: CapturedOutput) {
val first = "가".repeat(3332) + "\uD83D\uDE00"
val broken = "나".repeat(3333) + "b"
val source = first + broken + "synthetic-source-marker ${UUID.randomUUID()}"
val request = TranslateRequest(listOf(source, "second original"), "ko", "en")
failTranslationAfter(first)
val fallback = RecordingTranslationProvider(PAPAGO) { TranslateResult(listOf("papago first", "papago second")) }
val awsFirst = TranslationCoordinator(
listOf(fallback, awsTranslation),
LanguageProviderProperties(listOf(PAPAGO, AWS), listOf(AWS, PAPAGO)),
memoryRepository
)
val result = awsFirst.translate(request)
assertEquals(PAPAGO, result.providerId)
assertEquals("PAPAGO-v1", result.providerVersion)
assertEquals(listOf("papago first", "papago second"), result.result.translatedText)
assertSame(request, fallback.requests.single())
assertTranslationRequests(first, broken)
assertEquals(0L, transaction.execute { memoryRepository.count() })
assertFalse(output.all.contains("synthetic-sensitive-sdk-marker"))
assertFalse(output.all.contains("synthetic-source-marker"))
}
private fun failTranslationAfter(first: String) {
Mockito.`when`(translateClient.translateText(Mockito.any(TranslateTextRequest::class.java))).thenAnswer {
if (it.getArgument<TranslateTextRequest>(0).text() == first) {
translationResponse("partial success")
} else {
throw SdkClientException.create("synthetic-sensitive-sdk-marker")
}
}
}
private fun detectionResponse(vararg codes: String): DetectDominantLanguageResponse {
return DetectDominantLanguageResponse.builder()
.languages(codes.map { DominantLanguage.builder().languageCode(it).score(0.9f).build() })
.build()
}
private fun translationResponse(text: String): TranslateTextResponse {
return TranslateTextResponse.builder().sourceLanguageCode("ko").targetLanguageCode("en").translatedText(text).build()
}
private fun assertDetectionRequests(vararg texts: String) {
val captor = ArgumentCaptor.forClass(DetectDominantLanguageRequest::class.java)
Mockito.verify(comprehendClient, Mockito.times(texts.size)).detectDominantLanguage(captor.capture())
assertEquals(texts.toList(), captor.allValues.map { it.text() })
Mockito.verifyNoMoreInteractions(comprehendClient)
}
private fun assertTranslationRequests(vararg texts: String) {
val captor = ArgumentCaptor.forClass(TranslateTextRequest::class.java)
Mockito.verify(translateClient, Mockito.times(texts.size)).translateText(captor.capture())
assertEquals(texts.toList(), captor.allValues.map { it.text() })
captor.allValues.forEach {
assertEquals("ko", it.sourceLanguageCode())
assertEquals("en", it.targetLanguageCode())
assertTrue(it.text().toByteArray(Charsets.UTF_8).size <= 10000)
}
Mockito.verifyNoMoreInteractions(translateClient)
}
private fun schedule(postId: Long, source: String) {
@@ -163,21 +338,18 @@ class LanguageProviderCacheEndToEndTest @Autowired constructor(
@TestConfiguration
class Providers {
@Bean
fun languageProviderProperties() = LanguageProviderProperties(
listOf(PAPAGO, AWS),
listOf(PAPAGO, AWS)
)
fun languageProviderProperties() = LanguageProviderProperties()
@Bean
fun papagoDetection() = RecordingDetectionProvider(PAPAGO) { throw IllegalStateException("sensitive provider body") }
@Bean
fun secondaryDetection() = RecordingDetectionProvider(AWS) { "ko" }
fun comprehendClient(): ComprehendClient = Mockito.mock(ComprehendClient::class.java)
@Bean
fun papagoTranslation() = RecordingTranslationProvider(PAPAGO) { TranslateResult(emptyList()) }
@Bean
fun secondaryTranslation() = RecordingTranslationProvider(AWS)
fun translateClient(): TranslateClient = Mockito.mock(TranslateClient::class.java)
}
}