fix(i18n): 콘텐츠 태그 미번역을 Papago로 전환하고 완료 작업을 복구한다

This commit is contained in:
2026-11-17 09:56:19 +09:00
parent cd4667813a
commit 55669c4465
19 changed files with 742 additions and 24 deletions
@@ -53,18 +53,34 @@ class ResourceTranslationJobScheduler(
resourceIds,
target,
TranslationJobStatus.FAILED
)
) + if (resourceType == LanguageTranslationTargetType.CONTENT) {
translationJobRepository.findCompletedContentTagJobs(resourceIds, target)
} else {
emptyList()
}
val now = LocalDateTime.now()
candidates.groupBy { it.resourceId }.forEach { (resourceId, failedJobs) ->
candidates.groupBy { it.resourceId }.forEach { (resourceId, resourceJobs) ->
val source = sourceExtractor.extract(resourceType, resourceId) ?: return@forEach
val sourceLanguage = source.sourceLanguage.lowercase()
if (!TranslationLanguages.isSupported(sourceLanguage) || sourceLanguage == target) return@forEach
val sourceHashes = source.fields.associate { it.fieldKey to SourceTextNormalizer.hash(it.sourceText) }
failedJobs.filter { job ->
job.sourceLanguage == sourceLanguage && job.sourceHash == sourceHashes[job.fieldKey]
val sourceFields = source.fields.associateBy { it.fieldKey }
resourceJobs.filter { job ->
job.sourceLanguage == sourceLanguage &&
job.sourceHash == sourceFields[job.fieldKey]?.let { SourceTextNormalizer.hash(it.sourceText) }
}.forEach { job ->
translationJobRepository.restartFailedJob(requireNotNull(job.id), now)
when (job.status) {
TranslationJobStatus.FAILED -> translationJobRepository.restartFailedJob(requireNotNull(job.id), now)
TranslationJobStatus.COMPLETED -> translationJobScheduler.scheduleMissingTranslation(
resourceType = resourceType,
resourceId = resourceId,
fieldKey = job.fieldKey,
sourceText = sourceFields.getValue(job.fieldKey).sourceText,
sourceLanguage = sourceLanguage,
targetLanguage = target
)
TranslationJobStatus.PENDING, TranslationJobStatus.RUNNING -> Unit
}
}
}
}
@@ -20,4 +20,8 @@ object SourceTextNormalizer {
.digest(normalized.toByteArray(Charsets.UTF_8))
return digest.joinToString("") { "%02x".format(it) }
}
fun hasSameCommaSeparatedText(first: String, second: String): Boolean {
return first.split(',').map(::normalize) == second.split(',').map(::normalize)
}
}
@@ -3,7 +3,8 @@ package kr.co.vividnext.sodalive.i18n.translation
data class TranslateRequest(
val texts: List<String>,
val sourceLanguage: String,
val targetLanguage: String
val targetLanguage: String,
val rejectUnchangedAwsResult: Boolean = false
)
data class TranslateResult(
@@ -10,7 +10,12 @@ class TranslationCoordinator(
) {
private val providers = properties.orderedProviders(properties.translationOrder, providers) { it.providerId }
fun findCached(sourceHash: String, sourceLanguage: String, targetLanguage: String): TranslationMemory? {
fun findCached(
sourceHash: String,
sourceLanguage: String,
targetLanguage: String,
rejectUnchangedAwsResult: Boolean = false
): TranslationMemory? {
providers.forEach { provider ->
val memory = translationMemoryRepository
.findBySourceHashAndSourceLanguageAndTargetLanguageAndProviderAndNormalizationVersion(
@@ -20,7 +25,14 @@ class TranslationCoordinator(
provider.providerId.code,
SourceTextNormalizer.NORMALIZATION_VERSION
)
if (memory != null && memory.translatedText.isNotBlank()) return memory
if (memory == null || memory.translatedText.isBlank()) return@forEach
if (
rejectUnchangedAwsResult && provider.providerId == LanguageProviderId.AWS &&
SourceTextNormalizer.hasSameCommaSeparatedText(memory.sourceText, memory.translatedText)
) {
return@forEach
}
return memory
}
return null
}
@@ -42,6 +54,14 @@ class TranslationCoordinator(
result != null && result.translatedText.size == request.texts.size &&
result.translatedText.none { it.isBlank() }
) {
if (
request.rejectUnchangedAwsResult && provider.providerId == LanguageProviderId.AWS &&
request.texts.zip(result.translatedText).any { (source, translated) ->
SourceTextNormalizer.hasSameCommaSeparatedText(source, translated)
}
) {
return@forEach
}
return ProviderTranslationResult(result, provider.providerId, provider.providerVersion)
}
}
@@ -22,6 +22,21 @@ interface TranslationJobRepository : JpaRepository<TranslationJob, Long> {
status: TranslationJobStatus
): List<TranslationJob>
@Query(
"""
select j from TranslationJob j
where j.resourceType = kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType.CONTENT
and j.resourceId in :resourceIds
and j.targetLanguage = :targetLanguage
and j.fieldKey = 'tags'
and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.COMPLETED
"""
)
fun findCompletedContentTagJobs(
@Param("resourceIds") resourceIds: Collection<Long>,
@Param("targetLanguage") targetLanguage: String
): List<TranslationJob>
@Modifying
@Query(
"""
@@ -36,6 +51,22 @@ interface TranslationJobRepository : JpaRepository<TranslationJob, Long> {
)
fun restartFailedJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int
@Modifying
@Query(
"""
update TranslationJob j
set j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.PENDING,
j.retryCount = 0,
j.lastErrorMessage = null,
j.nextRetryAt = :now
where j.id = :jobId
and j.resourceType = kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType.CONTENT
and j.fieldKey = 'tags'
and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.COMPLETED
"""
)
fun restartCompletedContentTagsJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int
@Query(
"""
select j from TranslationJob j
@@ -1,12 +1,15 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository
import org.springframework.stereotype.Service
import org.springframework.transaction.annotation.Transactional
import java.time.LocalDateTime
@Service
class TranslationJobScheduler(
private val translationJobRepository: TranslationJobRepository
private val translationJobRepository: TranslationJobRepository,
private val translationCoordinator: TranslationCoordinator,
private val contentTranslationRepository: ContentTranslationRepository
) {
@Transactional
fun scheduleMissingTranslation(
@@ -35,8 +38,34 @@ class TranslationJobScheduler(
sourceHash = sourceHash
)
if (existingJob != null) {
if (existingJob.status == TranslationJobStatus.FAILED && existingJob.sourceLanguage == normalizedSourceLanguage) {
translationJobRepository.restartFailedJob(existingJob.id!!, LocalDateTime.now())
if (existingJob.sourceLanguage != normalizedSourceLanguage) return
when (existingJob.status) {
TranslationJobStatus.FAILED ->
translationJobRepository.restartFailedJob(requireNotNull(existingJob.id), LocalDateTime.now())
TranslationJobStatus.COMPLETED -> {
if (resourceType != LanguageTranslationTargetType.CONTENT || fieldKey != "tags") return
val renderedTags = contentTranslationRepository.findByContentIdAndLocale(resourceId, normalizedTargetLanguage)
?.renderedPayload?.tags
if (
!renderedTags.isNullOrBlank() &&
!SourceTextNormalizer.hasSameCommaSeparatedText(normalizedText, renderedTags)
) {
return
}
val cached = translationCoordinator.findCached(
sourceHash,
normalizedSourceLanguage,
normalizedTargetLanguage,
rejectUnchangedAwsResult = true
)
if (cached != null && cached.translatedText == renderedTags) return
translationJobRepository.restartCompletedContentTagsJob(requireNotNull(existingJob.id), LocalDateTime.now())
}
TranslationJobStatus.PENDING, TranslationJobStatus.RUNNING -> Unit
}
return
}
@@ -69,14 +69,21 @@ class TranslationJobWorker(
}
private fun ensureMemory(job: TranslationJob) {
val existing = translationCoordinator.findCached(job.sourceHash, job.sourceLanguage, job.targetLanguage)
val rejectUnchangedAwsResult = job.resourceType == LanguageTranslationTargetType.CONTENT && job.fieldKey == "tags"
val existing = translationCoordinator.findCached(
job.sourceHash,
job.sourceLanguage,
job.targetLanguage,
rejectUnchangedAwsResult
)
if (existing != null) return
val response = translationCoordinator.translate(
TranslateRequest(
texts = listOf(job.sourceText),
sourceLanguage = job.sourceLanguage,
targetLanguage = job.targetLanguage
targetLanguage = job.targetLanguage,
rejectUnchangedAwsResult = rejectUnchangedAwsResult
)
)
val translated = response.result.translatedText.single()
@@ -102,6 +109,12 @@ class TranslationJobWorker(
providerVersion = response.providerVersion
)
)
} else if (
rejectUnchangedAwsResult && response.providerId == LanguageProviderId.AWS &&
SourceTextNormalizer.hasSameCommaSeparatedText(memory.sourceText, memory.translatedText)
) {
memory.translatedText = translated
translationMemoryRepository.save(memory)
}
}
}
@@ -30,7 +30,7 @@ class TranslationMemory(
val targetLanguage: String,
@Column(name = "translated_text", nullable = false, columnDefinition = "text")
val translatedText: String,
var translatedText: String,
@Column(name = "provider", nullable = false, length = 50)
val provider: String,
@@ -74,7 +74,9 @@ class TranslationReadModelMaterializer(
val memory = translationCoordinator.findCached(
sourceHash = SourceTextNormalizer.hash(normalizedText),
sourceLanguage = source.sourceLanguage.lowercase(),
targetLanguage = targetLanguage
targetLanguage = targetLanguage,
rejectUnchangedAwsResult = source.resourceType == LanguageTranslationTargetType.CONTENT &&
field.fieldKey == "tags"
) ?: return null
result[field.fieldKey] = memory.translatedText
@@ -16,6 +16,7 @@ import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.TranslationJob
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobSchedulerTestConfiguration
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus
import kr.co.vividnext.sodalive.i18n.translation.TranslationSourceExtractor
import kr.co.vividnext.sodalive.member.Member
@@ -56,7 +57,8 @@ import javax.persistence.EntityManager
LanguageDetectListener::class,
TranslationSourceExtractor::class,
ResourceTranslationJobScheduler::class,
TranslationJobScheduler::class
TranslationJobScheduler::class,
TranslationJobSchedulerTestConfiguration::class
)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
class MetadataLanguageDetectTransactionTest @Autowired constructor(
@@ -0,0 +1,273 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.content.translation.ContentTranslation
import kr.co.vividnext.sodalive.content.translation.ContentTranslationPayload
import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.CsvSource
import org.junit.jupiter.params.provider.EnumSource
import org.junit.jupiter.params.provider.ValueSource
import org.mockito.AdditionalAnswers
import org.mockito.Mockito
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.transaction.annotation.Propagation
import org.springframework.transaction.support.TransactionTemplate
import java.time.LocalDateTime
class CompletedContentTagRepairTest : MetadataReadRetryTestSupport() {
@Autowired
private lateinit var jobScheduler: TranslationJobScheduler
@Autowired
private lateinit var memories: TranslationMemoryRepository
@Autowired
private lateinit var translations: ContentTranslationRepository
@Autowired
private lateinit var coordinator: TranslationCoordinator
private val sourceText = "#비밀,#밤"
@ParameterizedTest
@CsvSource(
value = [
"#비밀,#밤|aws|#비밀,#밤",
"#비밀, #밤|aws|#비밀, #밤",
"#비밀, #밤|''|''",
"#비밀, #밤|papago|#비밀,#밤",
"#비밀, #밤|papago|#secret, #night",
"#비밀, #밤|aws|#secret, #night"
],
delimiter = '|'
)
@DisplayName("원문과 같은 완료 태그가 유효 캐시와 정확히 일치하지 않으면 같은 작업을 즉시 재개한다")
fun shouldRestartCompletedTagsWithoutMatchingAcceptedCache(renderedTags: String, provider: String, cachedText: String) {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val job = givenJob(source)
givenRenderedTags(renderedTags)
val memory = cachedText.takeIf { it.isNotEmpty() }?.let { givenCache(it, provider) }
val requestedAt = LocalDateTime.now().withNano(0)
schedule(source)
assertRepaired(job, requestedAt)
assertEquals(renderedTags, translations.findByContentIdAndLocale(10L, "en")?.renderedPayload?.tags)
assertEquals(listOfNotNull(memory?.id), memories.findAll().map { it.id })
}
@Test
@DisplayName("기존 번역 행이 있는 조회 재시작도 현재 COMPLETED 태그를 복구 후보로 검사한다")
fun shouldRestartCompletedTagsOnReadDespiteExistingRenderedRow() {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val job = givenJob(source)
givenRenderedTags("#비밀, #밤")
val memory = givenCache("#비밀, #밤")
val requestedAt = LocalDateTime.now().withNano(0)
scheduler.restartFailedResourceTranslations(source.resourceType, listOf(source.resourceId), "EN")
assertRepaired(job, requestedAt)
assertEquals(memory.id, memories.findAll().single().id)
assertEquals("#비밀, #밤", translations.findByContentIdAndLocale(10L, "en")?.renderedPayload?.tags)
}
@ParameterizedTest
@ValueSource(strings = ["#비밀,#밤", "#비밀, #밤"])
@DisplayName("잘못된 AWS 캐시가 남아 있어도 Papago로 확인된 동일 태그는 반복 수정과 조회에서 완료를 유지한다")
fun shouldPreservePapagoConfirmedIdenticalTagsAcrossRepeatedRequests(renderedTags: String) {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val job = givenJob(source)
givenRenderedTags(renderedTags)
val aws = givenCache("#비밀 , #밤")
val papago = givenCache(renderedTags, "papago")
repeat(2) {
schedule(source)
scheduler.restartFailedResourceTranslations(source.resourceType, listOf(source.resourceId), "en")
}
assertPreserved(job)
assertEquals(setOf(aws.id, papago.id), memories.findAll().map { it.id }.toSet())
assertEquals(renderedTags, translations.findByContentIdAndLocale(10L, "en")?.renderedPayload?.tags)
}
@ParameterizedTest
@ValueSource(booleans = [false, true])
@DisplayName("정상 번역된 완료 태그는 AWS 캐시 유무와 관계없이 수정과 조회에서 보존한다")
fun shouldPreserveNormallyTranslatedTags(withAwsCache: Boolean) {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val job = givenJob(source)
givenRenderedTags("#secret, #night")
if (withAwsCache) givenCache("#secret, #night")
schedule(source)
scheduler.restartFailedResourceTranslations(source.resourceType, listOf(source.resourceId), "en")
assertPreserved(job)
}
@ParameterizedTest
@CsvSource("CONTENT,title", "CONTENT,detail", "CHARACTER,tags", "ORIGINAL_WORK,tags", "SERIES,keywords")
@DisplayName("원문과 같아도 콘텐츠 태그 이외의 완료 필드와 리소스는 초기화하지 않는다")
fun shouldPreserveCompletedFieldsOutsideContentTags(type: LanguageTranslationTargetType, field: String) {
val source = givenSource(type, field, sourceText)
val job = givenJob(source)
givenRenderedTags("#비밀, #밤")
schedule(source)
scheduler.restartFailedResourceTranslations(type, listOf(source.resourceId), "en")
assertPreserved(job)
}
@ParameterizedTest
@EnumSource(TranslationJobStatus::class, names = ["PENDING", "RUNNING"])
@DisplayName("미번역 태그라도 진행 중 작업의 재시도 예산과 예약 시각은 유지한다")
fun shouldPreserveActiveTagJobs(status: TranslationJobStatus) {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val job = givenJob(source, status)
givenRenderedTags("#비밀, #밤")
schedule(source)
scheduler.restartFailedResourceTranslations(source.resourceType, listOf(source.resourceId), "en")
assertPreserved(job)
}
@ParameterizedTest
@CsvSource("false,language", "false,hash", "true,language", "true,hash")
@DisplayName("현재 원문 언어 또는 해시가 다른 완료 태그는 수정과 조회에서 재개하지 않는다")
fun shouldPreserveCompletedTagsForDifferentCurrentSource(readRequest: Boolean, mismatch: String) {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val previousSource = if (mismatch == "language") {
source.copy(sourceLanguage = "ja")
} else {
source.copy(fields = listOf(TranslationSourceField("tags", "#이전,#태그")))
}
val job = givenJob(previousSource)
givenRenderedTags("#비밀, #밤")
if (readRequest) {
scheduler.restartFailedResourceTranslations(source.resourceType, listOf(source.resourceId), "en")
} else {
schedule(source)
}
assertPreserved(job)
assertEquals(if (!readRequest && mismatch == "hash") 2L else 1L, jobs.count())
}
@ParameterizedTest
@CsvSource("false,false", "false,true", "true,false", "true,true")
@DisplayName("완료 태그의 저장 응답이 없거나 비어 있으면 캐시로 재생성할 수 있어도 작업을 재개한다")
fun shouldRestartCompletedTagsWithMissingRenderedPayload(blankTags: Boolean, withPapagoCache: Boolean) {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val job = givenJob(source)
if (blankTags) givenRenderedTags("")
if (withPapagoCache) givenCache(sourceText, "papago")
val requestedAt = LocalDateTime.now().withNano(0)
schedule(source)
assertRepaired(job, requestedAt)
}
@Test
@DisplayName("완료 작업 조회 뒤 다른 트랜잭션이 RUNNING으로 선점하면 오래된 복구 요청은 초기화하지 않는다")
fun shouldPreserveClaimCommittedAfterCompletedSnapshotWasRead() {
val source = givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
val job = givenJob(source)
givenRenderedTags("#비밀, #밤")
val repository = Mockito.mock(
TranslationJobRepository::class.java,
AdditionalAnswers.delegatesTo<TranslationJobRepository>(jobs)
)
val claimTransaction = TransactionTemplate(transactionManager).apply {
propagationBehavior = Propagation.REQUIRES_NEW.value()
}
Mockito.doAnswer {
val snapshot = jobs.findById(requireNotNull(job.id)).orElseThrow()
claimTransaction.executeWithoutResult {
val claimed = jobs.findById(requireNotNull(job.id)).orElseThrow()
claimed.status = TranslationJobStatus.RUNNING
claimed.retryCount = 4
jobs.saveAndFlush(claimed)
}
snapshot
}.`when`(repository).findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
source.resourceType,
source.resourceId,
"tags",
"en",
SourceTextNormalizer.hash(sourceText)
)
TransactionTemplate(transactionManager).executeWithoutResult {
schedule(source, TranslationJobScheduler(repository, coordinator, translations))
}
val claimed = jobs.findById(requireNotNull(job.id)).orElseThrow()
assertEquals(TranslationJobStatus.RUNNING, claimed.status)
assertEquals(4, claimed.retryCount)
assertEquals(job.lastErrorMessage, claimed.lastErrorMessage)
assertEquals(previousRetryAt, claimed.nextRetryAt)
}
private fun givenJob(
source: TranslationSource,
status: TranslationJobStatus = TranslationJobStatus.COMPLETED
): TranslationJob {
val job = failedJob(source)
job.status = status
return jobs.saveAndFlush(job)
}
private fun givenRenderedTags(tags: String) = persistRendered(
ContentTranslation(10L, "en", ContentTranslationPayload("Old title", "Old detail", tags)),
"content_translation",
"""{"title":"Old title","detail":"Old detail","tags":"$tags"}"""
)
private fun givenCache(text: String, provider: String = "aws") = persist(
TranslationMemory(SourceTextNormalizer.hash(sourceText), sourceText, "ko", "en", text, provider, "test")
)
private fun schedule(source: TranslationSource, scheduler: TranslationJobScheduler = jobScheduler) {
val field = source.fields.single()
scheduler.scheduleMissingTranslation(
source.resourceType,
source.resourceId,
field.fieldKey,
field.sourceText,
"KO",
"EN"
)
}
private fun assertRepaired(job: TranslationJob, requestedAt: LocalDateTime) {
val repaired = jobs.findAll().single()
assertEquals(job.id, repaired.id)
assertEquals(job.sourceHash, repaired.sourceHash)
assertEquals(job.sourceText, repaired.sourceText)
assertEquals(job.sourceLanguage, repaired.sourceLanguage)
assertEquals(TranslationJobStatus.PENDING, repaired.status)
assertEquals(0, repaired.retryCount)
assertNull(repaired.lastErrorMessage)
assertFalse(repaired.nextRetryAt.isBefore(requestedAt))
assertFalse(repaired.nextRetryAt.isAfter(LocalDateTime.now().plusSeconds(1)))
}
private fun assertPreserved(job: TranslationJob) {
val preserved = jobs.findById(requireNotNull(job.id)).orElseThrow()
assertEquals(job.status, preserved.status)
assertEquals(job.retryCount, preserved.retryCount)
assertEquals(job.lastErrorMessage, preserved.lastErrorMessage)
assertEquals(previousRetryAt, preserved.nextRetryAt)
}
}
@@ -0,0 +1,286 @@
package kr.co.vividnext.sodalive.i18n.translation
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import kr.co.vividnext.sodalive.content.translation.ContentTranslation
import kr.co.vividnext.sodalive.content.translation.ContentTranslationPayload
import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository
import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.ValueSource
import org.mockito.AdditionalAnswers
import org.mockito.Mockito
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.context.TestConfiguration
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Import
import org.springframework.jdbc.core.JdbcTemplate
import java.time.LocalDateTime
@Import(
TranslationCoordinator::class,
TranslationReadModelMaterializer::class,
ContentTagTranslationIntegrationTest.Providers::class
)
class ContentTagTranslationIntegrationTest : MetadataReadRetryTestSupport() {
@Autowired
private lateinit var coordinator: TranslationCoordinator
@Autowired
private lateinit var materializer: TranslationReadModelMaterializer
@Autowired
private lateinit var memories: TranslationMemoryRepository
@Autowired
private lateinit var translations: ContentTranslationRepository
@Autowired
private lateinit var jdbc: JdbcTemplate
@Autowired
private lateinit var aws: TagProvider
@Autowired
private lateinit var papago: PapagoTagProvider
private val sourceText = "#힐링,#수면"
@BeforeEach
fun prepareProviders() {
aws.result = "#힐링, #수면"
aws.calls = 0
papago.result = "#Healing, #Sleep"
papago.calls = 0
papago.fail = false
}
@AfterEach
fun removeWorkerResults() {
translations.deleteAllInBatch()
memories.deleteAllInBatch()
}
@ParameterizedTest
@ValueSource(strings = ["#힐링,#수면", "#힐링, #수면", " #힐링 , #수면 "])
@DisplayName("AWS가 콘텐츠 태그 원문이나 쉼표 공백만 바꾸면 Papago로 번역하고 저장한다")
fun shouldFallbackForUnchangedAwsContentTags(awsResult: String) {
aws.result = awsResult
prepareJob()
assertTrue(worker().processNextGroup())
assertEquals(1, aws.calls)
assertEquals(1, papago.calls)
assertEquals("papago", memories.findAll().single().provider)
assertEquals("#Healing, #Sleep", renderedTags())
assertEquals(TranslationJobStatus.COMPLETED, jobs.findAll().single().status)
}
@Test
@DisplayName("이미 저장된 AWS 미번역 태그 캐시도 건너뛰고 Papago 결과로 응답 데이터를 조립한다")
fun shouldIgnoreUnchangedAwsMemoryDuringWorkerAndMaterialization() {
prepareJob()
memories.saveAndFlush(
TranslationMemory(SourceTextNormalizer.hash(sourceText), sourceText, "ko", "en", aws.result, "aws", "test")
)
assertTrue(worker().processNextGroup())
assertEquals(1, papago.calls)
assertEquals(setOf("aws", "papago"), memories.findAll().map { it.provider }.toSet())
assertEquals("#Healing, #Sleep", renderedTags())
assertEquals(TranslationJobStatus.COMPLETED, jobs.findAll().single().status)
}
@Test
@DisplayName("기존 AWS 미번역 캐시 뒤 정상 AWS 결과를 얻으면 같은 캐시 행을 교체하고 응답을 조립한다")
fun shouldReplaceUnchangedAwsMemoryWithChangedAwsResult() {
prepareJob()
val previous = memories.saveAndFlush(
TranslationMemory(SourceTextNormalizer.hash(sourceText), sourceText, "ko", "en", aws.result, "aws", "test")
)
aws.result = "#Healing,#Sleep"
assertTrue(worker().processNextGroup())
val replaced = memories.findAll().single()
assertEquals("#Healing,#Sleep", replaced.translatedText)
assertEquals(previous.id, replaced.id)
assertEquals(0, papago.calls)
assertEquals("#Healing,#Sleep", renderedTags())
assertEquals(TranslationJobStatus.COMPLETED, jobs.findAll().single().status)
}
@Test
@DisplayName("AWS 미번역 태그 캐시만 있을 때는 정상 번역 응답으로 조립하지 않는다")
fun shouldNotMaterializeUnchangedAwsTagCache() {
givenSource(LanguageTranslationTargetType.CONTENT, "tags", sourceText)
memories.saveAndFlush(
TranslationMemory(SourceTextNormalizer.hash(sourceText), sourceText, "ko", "en", aws.result, "aws", "test")
)
assertFalse(materializer.materialize(LanguageTranslationTargetType.CONTENT, 10L, "en"))
assertEquals(0L, translations.count())
}
@Test
@DisplayName("Papago도 태그 원문을 반환하면 승인된 정책에 따라 완료한다")
fun shouldCompleteWithUnchangedPapagoTags() {
papago.result = sourceText
prepareJob()
assertTrue(worker().processNextGroup())
assertEquals("papago", memories.findAll().single().provider)
assertEquals(sourceText, renderedTags())
assertEquals(TranslationJobStatus.COMPLETED, jobs.findAll().single().status)
}
@Test
@DisplayName("AWS가 원문 태그를 반환하고 Papago가 실패하면 완료하지 않고 기존 재시도 정책을 따른다")
fun shouldRetryWhenPapagoFailsAfterUnchangedAwsTags() {
papago.fail = true
prepareJob()
assertTrue(worker().processNextGroup())
assertEquals(1, papago.calls)
assertEquals(TranslationJobStatus.PENDING, jobs.findAll().single().status)
assertEquals(1, jobs.findAll().single().retryCount)
assertEquals(0L, memories.count())
assertEquals(0L, translations.count())
}
@Test
@DisplayName("이전 완료 태그는 조회로 복구하고 Papago 동일 결과 저장 후 다음 조회에서는 다시 등록하지 않는다")
fun shouldRepairCompletedTagsOnceAcrossReadWorkerAndRead() {
prepareJob()
val job = jobs.findAll().single().apply { status = TranslationJobStatus.COMPLETED }
jobs.saveAndFlush(job)
memories.saveAndFlush(
TranslationMemory(SourceTextNormalizer.hash(sourceText), sourceText, "ko", "en", aws.result, "aws", "test")
)
translations.saveAndFlush(ContentTranslation(10L, "en", ContentTranslationPayload("", "", aws.result)))
storeRenderedPayloadAsJsonObject()
papago.result = sourceText
scheduler.restartFailedResourceTranslations(LanguageTranslationTargetType.CONTENT, listOf(10L), "en")
assertEquals(TranslationJobStatus.PENDING, jobs.findAll().single().status)
assertTrue(worker().processNextGroup())
assertEquals(sourceText, renderedTags())
storeRenderedPayloadAsJsonObject()
scheduler.restartFailedResourceTranslations(LanguageTranslationTargetType.CONTENT, listOf(10L), "en")
val completed = jobs.findAll().single()
assertEquals(job.id, completed.id)
assertEquals(TranslationJobStatus.COMPLETED, completed.status)
assertEquals(1, papago.calls)
assertFalse(worker().processNextGroup())
println(
"Tag repair QA: same job=${job.id}, AWS unchanged rejected, " +
"Papago identical accepted, next read remains COMPLETED"
)
}
@Test
@DisplayName("AWS가 실제로 다른 태그를 반환하면 Papago를 호출하지 않는다")
fun shouldKeepChangedAwsTags() {
aws.result = "#Healing,#Sleep"
prepareJob()
assertTrue(worker().processNextGroup())
assertEquals(0, papago.calls)
assertEquals("aws", memories.findAll().single().provider)
assertEquals("#Healing,#Sleep", renderedTags())
}
@Test
@DisplayName("콘텐츠 제목의 원문 동일 결과에는 태그 전용 거부 정책을 적용하지 않는다")
fun shouldPreserveUnchangedTitlePolicy() {
aws.result = sourceText
prepareJob("title")
assertTrue(worker().processNextGroup())
assertEquals(0, papago.calls)
assertEquals("aws", memories.findAll().single().provider)
assertEquals(TranslationJobStatus.COMPLETED, jobs.findAll().single().status)
}
private fun prepareJob(fieldKey: String = "tags") {
givenSource(LanguageTranslationTargetType.CONTENT, fieldKey, sourceText)
scheduler.scheduleResourceTranslation(LanguageTranslationTargetType.CONTENT, 10L, "en")
}
private fun worker(): TranslationJobWorker {
val claimRepository = Mockito.mock(
TranslationJobRepository::class.java,
AdditionalAnswers.delegatesTo<TranslationJobRepository>(jobs)
)
Mockito.doAnswer {
jobs.findAll().filter { it.status == TranslationJobStatus.PENDING }.mapNotNull { it.id }
}.`when`(claimRepository).findNextPendingGroupJobIdsForUpdate(
Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now()
)
return TranslationJobWorker(claimRepository, memories, coordinator, materializer, transactionManager)
}
private fun renderedTags(): String = renderedPayload().get("tags").asText()
private fun storeRenderedPayloadAsJsonObject() {
jdbc.update(
"update content_translation set rendered_payload = ? format json where content_id = 10",
renderedPayload().toString()
)
}
private fun renderedPayload(): JsonNode {
val raw = jdbc.queryForObject(
"select rendered_payload from content_translation where content_id = 10",
String::class.java
)!!
val mapper = jacksonObjectMapper()
val value = mapper.readTree(raw)
// H2 JSON은 converter가 저장한 JSON 문자열을 한 겹 감싸 반환한다.
return if (value.isTextual) mapper.readTree(value.asText()) else value
}
@TestConfiguration
class Providers {
@Bean
fun properties() = LanguageProviderProperties()
@Bean
fun aws() = TagProvider()
@Bean
fun papago() = PapagoTagProvider()
}
open class TagProvider : TranslationProvider {
override val providerId = LanguageProviderId.AWS
override val providerVersion = "test"
var result = ""
var calls = 0
var fail = false
override fun translate(request: TranslateRequest): TranslateResult {
calls += 1
if (fail) throw IllegalStateException("test provider failure")
return TranslateResult(request.texts.map { result })
}
}
class PapagoTagProvider : TagProvider() {
override val providerId = LanguageProviderId.PAPAGO
}
}
@@ -27,7 +27,12 @@ import javax.persistence.EntityManager
]
)
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
@Import(QueryDslConfig::class, ResourceTranslationJobScheduler::class, TranslationJobScheduler::class)
@Import(
QueryDslConfig::class,
ResourceTranslationJobScheduler::class,
TranslationJobScheduler::class,
TranslationJobSchedulerTestConfiguration::class
)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
abstract class MetadataReadRetryTestSupport {
@Autowired
@@ -51,6 +51,7 @@ import javax.persistence.EntityManager
TranslationSourceExtractor::class,
ResourceTranslationJobScheduler::class,
TranslationJobScheduler::class,
TranslationJobSchedulerTestConfiguration::class,
LanguageTranslationListener::class,
MetadataTranslationHttpIntegrationTest.Configuration::class
)
@@ -1,6 +1,7 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.configs.QueryDslConfig
import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNull
@@ -34,10 +35,12 @@ import javax.persistence.EntityManager
]
)
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
@Import(QueryDslConfig::class, TranslationJobScheduler::class)
@Import(QueryDslConfig::class, TranslationJobScheduler::class, TranslationJobSchedulerTestConfiguration::class)
class TranslationJobRetryIntegrationTest @Autowired constructor(
private val repository: TranslationJobRepository,
private val scheduler: TranslationJobScheduler,
private val coordinator: TranslationCoordinator,
private val contentTranslations: ContentTranslationRepository,
private val entityManager: EntityManager,
private val transactionManager: PlatformTransactionManager
) {
@@ -246,7 +249,7 @@ class TranslationJobRetryIntegrationTest @Autowired constructor(
)
try {
transaction.executeWithoutResult {
schedule(TranslationJobScheduler(interleavingRepository))
schedule(TranslationJobScheduler(interleavingRepository, coordinator, contentTranslations))
}
val claimed = repository.findAll().single()
@@ -1,5 +1,6 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Test
@@ -7,10 +8,13 @@ import org.mockito.ArgumentCaptor
import org.mockito.Mockito
class TranslationJobSchedulerTest {
private val coordinator = Mockito.mock(TranslationCoordinator::class.java)
private val contentTranslations = Mockito.mock(ContentTranslationRepository::class.java)
@Test
fun shouldCreateOnePendingJobForMissingNormalizedText() {
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
val scheduler = TranslationJobScheduler(jobRepository)
val scheduler = TranslationJobScheduler(jobRepository, coordinator, contentTranslations)
Mockito.`when`(
jobRepository.findActiveJob(
@@ -49,7 +53,7 @@ class TranslationJobSchedulerTest {
@Test
fun shouldNotCreateDuplicateJobWhenSameCompletedJobAlreadyExists() {
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
val scheduler = TranslationJobScheduler(jobRepository)
val scheduler = TranslationJobScheduler(jobRepository, coordinator, contentTranslations)
val sourceHash = SourceTextNormalizer.hash("제목")
Mockito.`when`(
@@ -88,7 +92,7 @@ class TranslationJobSchedulerTest {
@Test
fun shouldNotCreateJobWhenSourceLanguageIsUnsupported() {
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
val scheduler = TranslationJobScheduler(jobRepository)
val scheduler = TranslationJobScheduler(jobRepository, coordinator, contentTranslations)
scheduler.scheduleMissingTranslation(
resourceType = LanguageTranslationTargetType.CONTENT,
@@ -0,0 +1,22 @@
package kr.co.vividnext.sodalive.i18n.translation
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.test.context.TestConfiguration
import org.springframework.context.annotation.Bean
@TestConfiguration
class TranslationJobSchedulerTestConfiguration {
@Bean
@ConditionalOnMissingBean(TranslationCoordinator::class)
fun translationCoordinator(memories: TranslationMemoryRepository): TranslationCoordinator = TranslationCoordinator(
listOf(LanguageProviderId.AWS, LanguageProviderId.PAPAGO).map { id ->
object : TranslationProvider {
override val providerId = id
override val providerVersion = "test"
override fun translate(request: TranslateRequest): TranslateResult = error("unexpected paid translation")
}
},
LanguageProviderProperties(),
memories
)
}
@@ -28,6 +28,7 @@ import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType
import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobSchedulerTestConfiguration
import kr.co.vividnext.sodalive.i18n.translation.TranslationSourceExtractor
import kr.co.vividnext.sodalive.member.Member
import kr.co.vividnext.sodalive.member.MemberRole
@@ -61,6 +62,7 @@ import javax.persistence.EntityManager
ResourceTranslationJobScheduler::class,
TranslationSourceExtractor::class,
TranslationJobScheduler::class,
TranslationJobSchedulerTestConfiguration::class,
AudioContentThemeQueryRepository::class
)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
@@ -729,7 +729,11 @@ class CreatorCommunityTranslationServiceTest @Autowired constructor(
materializer = persistentMaterializer(sourceExtractor),
resourceTranslationJobScheduler = ResourceTranslationJobScheduler(
sourceExtractor,
TranslationJobScheduler(translationJobRepository),
TranslationJobScheduler(
translationJobRepository,
Mockito.mock(TranslationCoordinator::class.java),
Mockito.mock(ContentTranslationRepository::class.java)
),
translationJobRepository
),
applicationEventPublisher = applicationEventPublisher,