fix(i18n): 번역 작업을 1초 주기로 처리하고 실패 작업을 재개한다

This commit is contained in:
2026-11-16 21:46:24 +09:00
parent ed37990e2b
commit 3ed79f2acb
5 changed files with 382 additions and 13 deletions
@@ -1,6 +1,7 @@
package kr.co.vividnext.sodalive.i18n.translation package kr.co.vividnext.sodalive.i18n.translation
import org.springframework.data.jpa.repository.JpaRepository import org.springframework.data.jpa.repository.JpaRepository
import org.springframework.data.jpa.repository.Modifying
import org.springframework.data.jpa.repository.Query import org.springframework.data.jpa.repository.Query
import org.springframework.data.repository.query.Param import org.springframework.data.repository.query.Param
import java.time.LocalDateTime import java.time.LocalDateTime
@@ -14,6 +15,27 @@ interface TranslationJobRepository : JpaRepository<TranslationJob, Long> {
sourceHash: String sourceHash: String
): TranslationJob? ): TranslationJob?
fun findByResourceTypeAndResourceIdInAndTargetLanguageAndStatus(
resourceType: LanguageTranslationTargetType,
resourceIds: Collection<Long>,
targetLanguage: String,
status: TranslationJobStatus
): List<TranslationJob>
@Modifying
@Query(
"""
update TranslationJob j
set j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.PENDING,
j.retryCount = 0,
j.lastErrorMessage = null,
j.nextRetryAt = :now
where j.id = :jobId
and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.FAILED
"""
)
fun restartFailedJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int
@Query( @Query(
""" """
select j from TranslationJob j select j from TranslationJob j
@@ -34,7 +34,12 @@ class TranslationJobScheduler(
targetLanguage = normalizedTargetLanguage, targetLanguage = normalizedTargetLanguage,
sourceHash = sourceHash sourceHash = sourceHash
) )
if (existingJob != null) return if (existingJob != null) {
if (existingJob.status == TranslationJobStatus.FAILED && existingJob.sourceLanguage == normalizedSourceLanguage) {
translationJobRepository.restartFailedJob(existingJob.id!!, LocalDateTime.now())
}
return
}
translationJobRepository.save( translationJobRepository.save(
TranslationJob( TranslationJob(
@@ -18,7 +18,7 @@ class TranslationJobWorker(
private val log = LoggerFactory.getLogger(javaClass) private val log = LoggerFactory.getLogger(javaClass)
private val transactionTemplate = TransactionTemplate(transactionManager) private val transactionTemplate = TransactionTemplate(transactionManager)
@Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:600000}") @Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:1000}")
fun runPendingJobs() { fun runPendingJobs() {
repeat(MAX_GROUPS_PER_TICK) { repeat(MAX_GROUPS_PER_TICK) {
if (!processNextGroup()) return if (!processNextGroup()) return
@@ -121,7 +121,7 @@ class TranslationJobWorker(
val job = translationJobRepository.findById(jobId).orElse(null) ?: return@executeWithoutResult val job = translationJobRepository.findById(jobId).orElse(null) ?: return@executeWithoutResult
job.retryCount += 1 job.retryCount += 1
job.lastErrorMessage = ex.message?.take(MAX_ERROR_LENGTH) job.lastErrorMessage = ex.message?.take(MAX_ERROR_LENGTH)
if (job.retryCount >= MAX_RETRY_COUNT) { if (job.retryCount >= MAX_ATTEMPTS) {
job.status = TranslationJobStatus.FAILED job.status = TranslationJobStatus.FAILED
} else { } else {
job.status = TranslationJobStatus.PENDING job.status = TranslationJobStatus.PENDING
@@ -142,6 +142,6 @@ class TranslationJobWorker(
companion object { companion object {
private const val MAX_GROUPS_PER_TICK = 5 private const val MAX_GROUPS_PER_TICK = 5
private const val MAX_ERROR_LENGTH = 1000 private const val MAX_ERROR_LENGTH = 1000
private const val MAX_RETRY_COUNT = 3 private const val MAX_ATTEMPTS = 6
} }
} }
@@ -0,0 +1,294 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.configs.QueryDslConfig
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.EnumSource
import org.junit.jupiter.params.provider.ValueSource
import org.mockito.AdditionalAnswers
import org.mockito.Mockito
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
import org.springframework.context.annotation.Import
import org.springframework.data.jpa.repository.Query
import org.springframework.transaction.PlatformTransactionManager
import org.springframework.transaction.annotation.Propagation
import org.springframework.transaction.annotation.Transactional
import org.springframework.transaction.support.TransactionTemplate
import java.time.LocalDateTime
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import javax.persistence.EntityManager
@DataJpaTest(
properties = [
"spring.cache.type=none",
"spring.datasource.url=jdbc:h2:mem:translation-job-retry;MODE=MySQL;NON_KEYWORDS=VALUE;DB_CLOSE_ON_EXIT=FALSE"
]
)
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
@Import(QueryDslConfig::class, TranslationJobScheduler::class)
class TranslationJobRetryIntegrationTest @Autowired constructor(
private val repository: TranslationJobRepository,
private val scheduler: TranslationJobScheduler,
private val entityManager: EntityManager,
private val transactionManager: PlatformTransactionManager
) {
private val transaction = TransactionTemplate(transactionManager)
private val requiresNewTransaction = TransactionTemplate(transactionManager).apply {
propagationBehavior = Propagation.REQUIRES_NEW.value()
}
private val previousRetryAt = LocalDateTime.of(2040, 1, 1, 0, 0)
@ParameterizedTest
@ValueSource(ints = [3, 6])
@DisplayName("기존 실패 횟수가 3회 또는 6회인 FAILED 행을 같은 원문 요청으로 즉시 재개한다")
fun shouldResetSameFailedRowForCurrentNormalizedSource(previousFailures: Int) {
val original = persistJob(retryCount = previousFailures)
val requestStartedAt = LocalDateTime.now().withNano(0)
schedule()
val requestFinishedAt = LocalDateTime.now().plusSeconds(1).withNano(0)
entityManager.flush()
entityManager.clear()
val resumed = repository.findAll().single()
assertEquals(original.id, resumed.id)
assertEquals("제목", resumed.sourceText)
assertEquals(SourceTextNormalizer.hash("제목"), resumed.sourceHash)
assertEquals(TranslationJobStatus.PENDING, resumed.status)
assertEquals(0, resumed.retryCount)
assertNull(resumed.lastErrorMessage)
assertTrue(!resumed.nextRetryAt.isBefore(requestStartedAt))
assertTrue(resumed.nextRetryAt.isBefore(requestFinishedAt))
}
@ParameterizedTest
@EnumSource(TranslationJobStatus::class, names = ["PENDING", "RUNNING", "COMPLETED"])
@DisplayName("반복 요청은 진행 중이거나 완료된 행의 상태와 재시도 예산을 바꾸지 않는다")
fun shouldPreserveExistingRowOutsideFailedStatus(status: TranslationJobStatus) {
val original = persistJob(status = status, retryCount = 4)
repeat(2) { schedule() }
entityManager.flush()
entityManager.clear()
val preserved = repository.findAll().single()
assertEquals(original.id, preserved.id)
assertEquals(status, preserved.status)
assertEquals(4, preserved.retryCount)
assertEquals("previous failure", preserved.lastErrorMessage)
assertEquals(previousRetryAt, preserved.nextRetryAt)
}
@Test
@DisplayName("같은 원문 해시라도 원문 언어가 다른 FAILED 행은 재개하지 않는다")
fun shouldNotRestartFailedJobForDifferentSourceLanguage() {
val original = persistJob(sourceLanguage = "ja")
schedule()
entityManager.flush()
entityManager.clear()
val preserved = repository.findAll().single()
assertEquals(original.id, preserved.id)
assertEquals("ja", preserved.sourceLanguage)
assertEquals(TranslationJobStatus.FAILED, preserved.status)
assertEquals(6, preserved.retryCount)
assertEquals("previous failure", preserved.lastErrorMessage)
assertEquals(previousRetryAt, preserved.nextRetryAt)
}
@Test
@DisplayName("현재 원문 요청은 이전 원문의 FAILED 행을 재개하지 않고 현재 해시의 작업을 생성한다")
fun shouldLeaveFailedRowForPreviousSourceUntouched() {
val previous = persistJob(sourceText = "이전 제목")
schedule()
entityManager.flush()
entityManager.clear()
val jobs = repository.findAll()
assertEquals(2, jobs.size)
val unchanged = jobs.single { it.id == previous.id }
assertEquals(TranslationJobStatus.FAILED, unchanged.status)
assertEquals(6, unchanged.retryCount)
assertEquals("previous failure", unchanged.lastErrorMessage)
assertEquals(previousRetryAt, unchanged.nextRetryAt)
val current = jobs.single { it.sourceHash == SourceTextNormalizer.hash("제목") }
assertEquals(TranslationJobStatus.PENDING, current.status)
assertEquals(0, current.retryCount)
assertNull(current.lastErrorMessage)
}
@Test
@DisplayName("재시도 시각이 지난 FAILED 행도 요청 없이 워커가 자동 선점하지 않는다")
fun shouldNotPollFailedRowWithoutNewRequest() {
val original = persistJob().apply { nextRetryAt = LocalDateTime.of(2000, 1, 1, 0, 0) }
entityManager.flush()
val claimSql = TranslationJobRepository::class.java
.getMethod("findNextPendingGroupJobIdsForUpdate", LocalDateTime::class.java)
.getAnnotation(Query::class.java).value
.replace("skip locked", "") // H2에서는 잠금 옵션만 제외하고 실제 운영 쿼리의 선점 조건을 실행한다.
val claimRepository = Mockito.mock(
TranslationJobRepository::class.java,
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
)
Mockito.doAnswer { call ->
entityManager.createNativeQuery(claimSql)
.setParameter("now", call.getArgument<LocalDateTime>(0))
.resultList.map { (it as Number).toLong() }
}.`when`(claimRepository).findNextPendingGroupJobIdsForUpdate(
Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now()
)
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
val provider = object : TranslationProvider {
override val providerId = LanguageProviderId.PAPAGO
override val providerVersion = "test"
override fun translate(request: TranslateRequest): TranslateResult =
throw IllegalStateException("unexpected translation")
}
val worker = TranslationJobWorker(
claimRepository,
memoryRepository,
TranslationCoordinator(
listOf(provider),
LanguageProviderProperties(listOf(LanguageProviderId.PAPAGO), listOf(LanguageProviderId.PAPAGO)),
memoryRepository
),
Mockito.mock(TranslationReadModelMaterializer::class.java),
transactionManager
)
assertFalse(worker.processNextGroup())
entityManager.flush()
entityManager.clear()
val untouched = repository.findAll().single()
assertEquals(original.id, untouched.id)
assertEquals(TranslationJobStatus.FAILED, untouched.status)
assertEquals(6, untouched.retryCount)
assertEquals("previous failure", untouched.lastErrorMessage)
assertEquals(LocalDateTime.of(2000, 1, 1, 0, 0), untouched.nextRetryAt)
}
@Test
@Transactional(propagation = Propagation.NOT_SUPPORTED)
@DisplayName("같은 FAILED 행의 동시 재개 요청은 중복 행 없이 하나의 새 주기만 연다")
fun shouldResumeSameFailedRowFromConcurrentRequests() {
val original = persistJob()
val originalId = requireNotNull(original.id)
val executor = Executors.newFixedThreadPool(2)
val ready = CountDownLatch(2)
val start = CountDownLatch(1)
try {
val requests = (1..2).map {
executor.submit {
ready.countDown()
check(start.await(5, TimeUnit.SECONDS)) { "concurrent request start timed out" }
schedule()
}
}
assertTrue(ready.await(5, TimeUnit.SECONDS))
start.countDown()
requests.forEach { it.get(5, TimeUnit.SECONDS) }
val resumed = repository.findAll().single()
assertEquals(originalId, resumed.id)
assertEquals(TranslationJobStatus.PENDING, resumed.status)
assertEquals(0, resumed.retryCount)
assertNull(resumed.lastErrorMessage)
} finally {
start.countDown()
executor.shutdownNow()
check(executor.awaitTermination(5, TimeUnit.SECONDS)) { "concurrent requests did not stop" }
repository.deleteById(originalId)
}
}
@Test
@Transactional(propagation = Propagation.NOT_SUPPORTED)
@DisplayName("FAILED 조회 뒤 다른 트랜잭션이 RUNNING으로 선점하면 오래된 재개 요청은 예산을 초기화하지 않는다")
fun shouldPreserveClaimCommittedAfterFailedSnapshotWasRead() {
val originalId = requireNotNull(persistJob().id)
val interleavingRepository = Mockito.mock(
TranslationJobRepository::class.java,
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
)
Mockito.doAnswer {
val snapshot = repository.findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
LanguageTranslationTargetType.CONTENT,
10L,
"title",
"en",
SourceTextNormalizer.hash("제목")
)
requiresNewTransaction.executeWithoutResult {
val claimed = repository.findById(originalId).orElseThrow()
claimed.status = TranslationJobStatus.RUNNING
claimed.retryCount = 4
repository.saveAndFlush(claimed)
}
snapshot
}.`when`(interleavingRepository).findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
LanguageTranslationTargetType.CONTENT,
10L,
"title",
"en",
SourceTextNormalizer.hash("제목")
)
try {
transaction.executeWithoutResult {
schedule(TranslationJobScheduler(interleavingRepository))
}
val claimed = repository.findAll().single()
assertEquals(originalId, claimed.id)
assertEquals(TranslationJobStatus.RUNNING, claimed.status)
assertEquals(4, claimed.retryCount)
assertEquals("previous failure", claimed.lastErrorMessage)
assertEquals(previousRetryAt, claimed.nextRetryAt)
} finally {
repository.deleteById(originalId)
}
}
private fun schedule(scheduler: TranslationJobScheduler = this.scheduler) {
scheduler.scheduleMissingTranslation(
resourceType = LanguageTranslationTargetType.CONTENT,
resourceId = 10L,
fieldKey = "title",
sourceText = " 제목\n",
sourceLanguage = "ko",
targetLanguage = "EN"
)
}
private fun persistJob(
status: TranslationJobStatus = TranslationJobStatus.FAILED,
sourceText: String = "제목",
retryCount: Int = 6,
sourceLanguage: String = "ko"
): TranslationJob = repository.saveAndFlush(
TranslationJob(
resourceType = LanguageTranslationTargetType.CONTENT,
resourceId = 10L,
fieldKey = "title",
sourceHash = SourceTextNormalizer.hash(sourceText),
sourceText = sourceText,
sourceLanguage = sourceLanguage,
targetLanguage = "en",
status = status,
retryCount = retryCount,
lastErrorMessage = "previous failure",
nextRetryAt = previousRetryAt
)
)
}
@@ -2,8 +2,12 @@ package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderId.PAPAGO import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderId.PAPAGO
import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.CsvSource
import org.mockito.ArgumentCaptor import org.mockito.ArgumentCaptor
import org.mockito.Mockito import org.mockito.Mockito
import org.springframework.scheduling.annotation.Scheduled import org.springframework.scheduling.annotation.Scheduled
@@ -14,12 +18,13 @@ import java.util.Optional
class TranslationJobWorkerTest { class TranslationJobWorkerTest {
@Test @Test
fun shouldRunEveryTenMinutesByDefault() { @DisplayName("번역 워커의 기본 실행 간격은 1초다")
fun shouldRunEverySecondByDefault() {
val scheduled = TranslationJobWorker::class.java val scheduled = TranslationJobWorker::class.java
.getDeclaredMethod("runPendingJobs") .getDeclaredMethod("runPendingJobs")
.getAnnotation(Scheduled::class.java) .getAnnotation(Scheduled::class.java)
assertEquals("\${sodalive.translation-job.fixed-delay-ms:600000}", scheduled.fixedDelayString) assertEquals("\${sodalive.translation-job.fixed-delay-ms:1000}", scheduled.fixedDelayString)
} }
@Test @Test
@@ -210,8 +215,14 @@ class TranslationJobWorkerTest {
assertEquals(setOf(1L, 2L, 3L, 4L, 5L), completedResourceIds) assertEquals(setOf(1L, 2L, 3L, 4L, 5L), completedResourceIds)
} }
@Test @ParameterizedTest(name = "기존 실패 {0}회 뒤 실패하면 {1}, 대기 {2}분")
fun shouldRetryFailedJobWithBackoff() { @CsvSource("0, PENDING, 1", "1, PENDING, 5", "2, PENDING, 15", "3, PENDING, 15", "4, PENDING, 15", "5, FAILED, 0")
@DisplayName("최초 시도와 재시도 5회의 실패 횟수를 누적하고 여섯 번째 실패에 종료한다")
fun shouldScheduleFiveRetriesBeforeFailingOnSixthAttempt(
failuresBeforeAttempt: Int,
expectedStatus: TranslationJobStatus,
delayMinutes: Long
) {
val jobRepository = Mockito.mock(TranslationJobRepository::class.java) val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java) val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
val provider = failingProvider() val provider = failingProvider()
@@ -229,17 +240,54 @@ class TranslationJobWorkerTest {
) )
val job = translationJob() val job = translationJob()
job.id = 200L job.id = 200L
val beforeRetryAt = job.nextRetryAt job.retryCount = failuresBeforeAttempt
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L)) Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job)) Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
worker.processNextGroup() val attemptStartedAt = LocalDateTime.now()
assertTrue(worker.processNextGroup())
val attemptFinishedAt = LocalDateTime.now()
assertEquals(TranslationJobStatus.PENDING, job.status) assertEquals(expectedStatus, job.status, "failure #${failuresBeforeAttempt + 1}")
assertEquals(1, job.retryCount) assertEquals(failuresBeforeAttempt + 1, job.retryCount)
assertEquals("all translation providers failed", job.lastErrorMessage) assertEquals("all translation providers failed", job.lastErrorMessage)
assertTrue(job.nextRetryAt.isAfter(beforeRetryAt)) if (expectedStatus == TranslationJobStatus.PENDING) {
assertTrue(!job.nextRetryAt.isBefore(attemptStartedAt.plusMinutes(delayMinutes)))
assertTrue(!job.nextRetryAt.isAfter(attemptFinishedAt.plusMinutes(delayMinutes)))
}
}
@Test
@DisplayName("다섯 번 실패한 작업도 여섯 번째 시도가 성공하면 완료된다")
fun shouldCompleteSixthAttemptAfterFiveFailures() {
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
val materializer = Mockito.mock(TranslationReadModelMaterializer::class.java)
val worker = TranslationJobWorker(
translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository,
translationCoordinator = TranslationCoordinator(
listOf(successfulProvider()),
LanguageProviderProperties(listOf(PAPAGO), listOf(PAPAGO)),
memoryRepository
),
materializer = materializer,
transactionManager = TestTransactionManager()
)
val job = translationJob().also {
it.id = 200L
it.retryCount = 5
it.lastErrorMessage = "previous failure"
}
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
assertTrue(worker.processNextGroup())
assertEquals(TranslationJobStatus.COMPLETED, job.status)
assertEquals(5, job.retryCount)
assertNull(job.lastErrorMessage)
} }
private fun translationJob( private fun translationJob(