From 3ed79f2acb01d19419795eb7e4c8f2958104e221 Mon Sep 17 00:00:00 2001 From: Klaus Date: Mon, 16 Nov 2026 21:46:24 +0900 Subject: [PATCH] =?UTF-8?q?fix(i18n):=20=EB=B2=88=EC=97=AD=20=EC=9E=91?= =?UTF-8?q?=EC=97=85=EC=9D=84=201=EC=B4=88=20=EC=A3=BC=EA=B8=B0=EB=A1=9C?= =?UTF-8?q?=20=EC=B2=98=EB=A6=AC=ED=95=98=EA=B3=A0=20=EC=8B=A4=ED=8C=A8=20?= =?UTF-8?q?=EC=9E=91=EC=97=85=EC=9D=84=20=EC=9E=AC=EA=B0=9C=ED=95=9C?= =?UTF-8?q?=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../translation/TranslationJobRepository.kt | 22 ++ .../translation/TranslationJobScheduler.kt | 7 +- .../i18n/translation/TranslationJobWorker.kt | 6 +- .../TranslationJobRetryIntegrationTest.kt | 294 ++++++++++++++++++ .../translation/TranslationJobWorkerTest.kt | 66 +++- 5 files changed, 382 insertions(+), 13 deletions(-) create mode 100644 src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRetryIntegrationTest.kt diff --git a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRepository.kt b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRepository.kt index 06ce0b09..d803f13d 100644 --- a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRepository.kt +++ b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRepository.kt @@ -1,6 +1,7 @@ package kr.co.vividnext.sodalive.i18n.translation import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query import org.springframework.data.repository.query.Param import java.time.LocalDateTime @@ -14,6 +15,27 @@ interface TranslationJobRepository : JpaRepository { sourceHash: String ): TranslationJob? + fun findByResourceTypeAndResourceIdInAndTargetLanguageAndStatus( + resourceType: LanguageTranslationTargetType, + resourceIds: Collection, + targetLanguage: String, + status: TranslationJobStatus + ): List + + @Modifying + @Query( + """ + update TranslationJob j + set j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.PENDING, + j.retryCount = 0, + j.lastErrorMessage = null, + j.nextRetryAt = :now + where j.id = :jobId + and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.FAILED + """ + ) + fun restartFailedJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int + @Query( """ select j from TranslationJob j diff --git a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobScheduler.kt b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobScheduler.kt index 5f8f849a..4b2c1ff7 100644 --- a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobScheduler.kt +++ b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobScheduler.kt @@ -34,7 +34,12 @@ class TranslationJobScheduler( targetLanguage = normalizedTargetLanguage, sourceHash = sourceHash ) - if (existingJob != null) return + if (existingJob != null) { + if (existingJob.status == TranslationJobStatus.FAILED && existingJob.sourceLanguage == normalizedSourceLanguage) { + translationJobRepository.restartFailedJob(existingJob.id!!, LocalDateTime.now()) + } + return + } translationJobRepository.save( TranslationJob( diff --git a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorker.kt b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorker.kt index 5ea1d74e..d1f17db0 100644 --- a/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorker.kt +++ b/src/main/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorker.kt @@ -18,7 +18,7 @@ class TranslationJobWorker( private val log = LoggerFactory.getLogger(javaClass) private val transactionTemplate = TransactionTemplate(transactionManager) - @Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:600000}") + @Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:1000}") fun runPendingJobs() { repeat(MAX_GROUPS_PER_TICK) { if (!processNextGroup()) return @@ -121,7 +121,7 @@ class TranslationJobWorker( val job = translationJobRepository.findById(jobId).orElse(null) ?: return@executeWithoutResult job.retryCount += 1 job.lastErrorMessage = ex.message?.take(MAX_ERROR_LENGTH) - if (job.retryCount >= MAX_RETRY_COUNT) { + if (job.retryCount >= MAX_ATTEMPTS) { job.status = TranslationJobStatus.FAILED } else { job.status = TranslationJobStatus.PENDING @@ -142,6 +142,6 @@ class TranslationJobWorker( companion object { private const val MAX_GROUPS_PER_TICK = 5 private const val MAX_ERROR_LENGTH = 1000 - private const val MAX_RETRY_COUNT = 3 + private const val MAX_ATTEMPTS = 6 } } diff --git a/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRetryIntegrationTest.kt b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRetryIntegrationTest.kt new file mode 100644 index 00000000..4764bd42 --- /dev/null +++ b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobRetryIntegrationTest.kt @@ -0,0 +1,294 @@ +package kr.co.vividnext.sodalive.i18n.translation + +import kr.co.vividnext.sodalive.configs.QueryDslConfig +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource +import org.junit.jupiter.params.provider.ValueSource +import org.mockito.AdditionalAnswers +import org.mockito.Mockito +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest +import org.springframework.context.annotation.Import +import org.springframework.data.jpa.repository.Query +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionTemplate +import java.time.LocalDateTime +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import javax.persistence.EntityManager + +@DataJpaTest( + properties = [ + "spring.cache.type=none", + "spring.datasource.url=jdbc:h2:mem:translation-job-retry;MODE=MySQL;NON_KEYWORDS=VALUE;DB_CLOSE_ON_EXIT=FALSE" + ] +) +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +@Import(QueryDslConfig::class, TranslationJobScheduler::class) +class TranslationJobRetryIntegrationTest @Autowired constructor( + private val repository: TranslationJobRepository, + private val scheduler: TranslationJobScheduler, + private val entityManager: EntityManager, + private val transactionManager: PlatformTransactionManager +) { + private val transaction = TransactionTemplate(transactionManager) + private val requiresNewTransaction = TransactionTemplate(transactionManager).apply { + propagationBehavior = Propagation.REQUIRES_NEW.value() + } + private val previousRetryAt = LocalDateTime.of(2040, 1, 1, 0, 0) + + @ParameterizedTest + @ValueSource(ints = [3, 6]) + @DisplayName("기존 실패 횟수가 3회 또는 6회인 FAILED 행을 같은 원문 요청으로 즉시 재개한다") + fun shouldResetSameFailedRowForCurrentNormalizedSource(previousFailures: Int) { + val original = persistJob(retryCount = previousFailures) + val requestStartedAt = LocalDateTime.now().withNano(0) + + schedule() + + val requestFinishedAt = LocalDateTime.now().plusSeconds(1).withNano(0) + entityManager.flush() + entityManager.clear() + val resumed = repository.findAll().single() + assertEquals(original.id, resumed.id) + assertEquals("제목", resumed.sourceText) + assertEquals(SourceTextNormalizer.hash("제목"), resumed.sourceHash) + assertEquals(TranslationJobStatus.PENDING, resumed.status) + assertEquals(0, resumed.retryCount) + assertNull(resumed.lastErrorMessage) + assertTrue(!resumed.nextRetryAt.isBefore(requestStartedAt)) + assertTrue(resumed.nextRetryAt.isBefore(requestFinishedAt)) + } + + @ParameterizedTest + @EnumSource(TranslationJobStatus::class, names = ["PENDING", "RUNNING", "COMPLETED"]) + @DisplayName("반복 요청은 진행 중이거나 완료된 행의 상태와 재시도 예산을 바꾸지 않는다") + fun shouldPreserveExistingRowOutsideFailedStatus(status: TranslationJobStatus) { + val original = persistJob(status = status, retryCount = 4) + + repeat(2) { schedule() } + + entityManager.flush() + entityManager.clear() + val preserved = repository.findAll().single() + assertEquals(original.id, preserved.id) + assertEquals(status, preserved.status) + assertEquals(4, preserved.retryCount) + assertEquals("previous failure", preserved.lastErrorMessage) + assertEquals(previousRetryAt, preserved.nextRetryAt) + } + + @Test + @DisplayName("같은 원문 해시라도 원문 언어가 다른 FAILED 행은 재개하지 않는다") + fun shouldNotRestartFailedJobForDifferentSourceLanguage() { + val original = persistJob(sourceLanguage = "ja") + + schedule() + + entityManager.flush() + entityManager.clear() + val preserved = repository.findAll().single() + assertEquals(original.id, preserved.id) + assertEquals("ja", preserved.sourceLanguage) + assertEquals(TranslationJobStatus.FAILED, preserved.status) + assertEquals(6, preserved.retryCount) + assertEquals("previous failure", preserved.lastErrorMessage) + assertEquals(previousRetryAt, preserved.nextRetryAt) + } + + @Test + @DisplayName("현재 원문 요청은 이전 원문의 FAILED 행을 재개하지 않고 현재 해시의 작업을 생성한다") + fun shouldLeaveFailedRowForPreviousSourceUntouched() { + val previous = persistJob(sourceText = "이전 제목") + + schedule() + + entityManager.flush() + entityManager.clear() + val jobs = repository.findAll() + assertEquals(2, jobs.size) + val unchanged = jobs.single { it.id == previous.id } + assertEquals(TranslationJobStatus.FAILED, unchanged.status) + assertEquals(6, unchanged.retryCount) + assertEquals("previous failure", unchanged.lastErrorMessage) + assertEquals(previousRetryAt, unchanged.nextRetryAt) + val current = jobs.single { it.sourceHash == SourceTextNormalizer.hash("제목") } + assertEquals(TranslationJobStatus.PENDING, current.status) + assertEquals(0, current.retryCount) + assertNull(current.lastErrorMessage) + } + + @Test + @DisplayName("재시도 시각이 지난 FAILED 행도 요청 없이 워커가 자동 선점하지 않는다") + fun shouldNotPollFailedRowWithoutNewRequest() { + val original = persistJob().apply { nextRetryAt = LocalDateTime.of(2000, 1, 1, 0, 0) } + entityManager.flush() + val claimSql = TranslationJobRepository::class.java + .getMethod("findNextPendingGroupJobIdsForUpdate", LocalDateTime::class.java) + .getAnnotation(Query::class.java).value + .replace("skip locked", "") // H2에서는 잠금 옵션만 제외하고 실제 운영 쿼리의 선점 조건을 실행한다. + val claimRepository = Mockito.mock( + TranslationJobRepository::class.java, + AdditionalAnswers.delegatesTo(repository) + ) + Mockito.doAnswer { call -> + entityManager.createNativeQuery(claimSql) + .setParameter("now", call.getArgument(0)) + .resultList.map { (it as Number).toLong() } + }.`when`(claimRepository).findNextPendingGroupJobIdsForUpdate( + Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now() + ) + val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java) + val provider = object : TranslationProvider { + override val providerId = LanguageProviderId.PAPAGO + override val providerVersion = "test" + override fun translate(request: TranslateRequest): TranslateResult = + throw IllegalStateException("unexpected translation") + } + val worker = TranslationJobWorker( + claimRepository, + memoryRepository, + TranslationCoordinator( + listOf(provider), + LanguageProviderProperties(listOf(LanguageProviderId.PAPAGO), listOf(LanguageProviderId.PAPAGO)), + memoryRepository + ), + Mockito.mock(TranslationReadModelMaterializer::class.java), + transactionManager + ) + + assertFalse(worker.processNextGroup()) + + entityManager.flush() + entityManager.clear() + val untouched = repository.findAll().single() + assertEquals(original.id, untouched.id) + assertEquals(TranslationJobStatus.FAILED, untouched.status) + assertEquals(6, untouched.retryCount) + assertEquals("previous failure", untouched.lastErrorMessage) + assertEquals(LocalDateTime.of(2000, 1, 1, 0, 0), untouched.nextRetryAt) + } + + @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) + @DisplayName("같은 FAILED 행의 동시 재개 요청은 중복 행 없이 하나의 새 주기만 연다") + fun shouldResumeSameFailedRowFromConcurrentRequests() { + val original = persistJob() + val originalId = requireNotNull(original.id) + val executor = Executors.newFixedThreadPool(2) + val ready = CountDownLatch(2) + val start = CountDownLatch(1) + try { + val requests = (1..2).map { + executor.submit { + ready.countDown() + check(start.await(5, TimeUnit.SECONDS)) { "concurrent request start timed out" } + schedule() + } + } + assertTrue(ready.await(5, TimeUnit.SECONDS)) + start.countDown() + requests.forEach { it.get(5, TimeUnit.SECONDS) } + + val resumed = repository.findAll().single() + assertEquals(originalId, resumed.id) + assertEquals(TranslationJobStatus.PENDING, resumed.status) + assertEquals(0, resumed.retryCount) + assertNull(resumed.lastErrorMessage) + } finally { + start.countDown() + executor.shutdownNow() + check(executor.awaitTermination(5, TimeUnit.SECONDS)) { "concurrent requests did not stop" } + repository.deleteById(originalId) + } + } + + @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) + @DisplayName("FAILED 조회 뒤 다른 트랜잭션이 RUNNING으로 선점하면 오래된 재개 요청은 예산을 초기화하지 않는다") + fun shouldPreserveClaimCommittedAfterFailedSnapshotWasRead() { + val originalId = requireNotNull(persistJob().id) + val interleavingRepository = Mockito.mock( + TranslationJobRepository::class.java, + AdditionalAnswers.delegatesTo(repository) + ) + Mockito.doAnswer { + val snapshot = repository.findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash( + LanguageTranslationTargetType.CONTENT, + 10L, + "title", + "en", + SourceTextNormalizer.hash("제목") + ) + requiresNewTransaction.executeWithoutResult { + val claimed = repository.findById(originalId).orElseThrow() + claimed.status = TranslationJobStatus.RUNNING + claimed.retryCount = 4 + repository.saveAndFlush(claimed) + } + snapshot + }.`when`(interleavingRepository).findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash( + LanguageTranslationTargetType.CONTENT, + 10L, + "title", + "en", + SourceTextNormalizer.hash("제목") + ) + try { + transaction.executeWithoutResult { + schedule(TranslationJobScheduler(interleavingRepository)) + } + + val claimed = repository.findAll().single() + assertEquals(originalId, claimed.id) + assertEquals(TranslationJobStatus.RUNNING, claimed.status) + assertEquals(4, claimed.retryCount) + assertEquals("previous failure", claimed.lastErrorMessage) + assertEquals(previousRetryAt, claimed.nextRetryAt) + } finally { + repository.deleteById(originalId) + } + } + + private fun schedule(scheduler: TranslationJobScheduler = this.scheduler) { + scheduler.scheduleMissingTranslation( + resourceType = LanguageTranslationTargetType.CONTENT, + resourceId = 10L, + fieldKey = "title", + sourceText = " 제목\n", + sourceLanguage = "ko", + targetLanguage = "EN" + ) + } + + private fun persistJob( + status: TranslationJobStatus = TranslationJobStatus.FAILED, + sourceText: String = "제목", + retryCount: Int = 6, + sourceLanguage: String = "ko" + ): TranslationJob = repository.saveAndFlush( + TranslationJob( + resourceType = LanguageTranslationTargetType.CONTENT, + resourceId = 10L, + fieldKey = "title", + sourceHash = SourceTextNormalizer.hash(sourceText), + sourceText = sourceText, + sourceLanguage = sourceLanguage, + targetLanguage = "en", + status = status, + retryCount = retryCount, + lastErrorMessage = "previous failure", + nextRetryAt = previousRetryAt + ) + ) +} diff --git a/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorkerTest.kt b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorkerTest.kt index f5bb3916..488d1178 100644 --- a/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorkerTest.kt +++ b/src/test/kotlin/kr/co/vividnext/sodalive/i18n/translation/TranslationJobWorkerTest.kt @@ -2,8 +2,12 @@ package kr.co.vividnext.sodalive.i18n.translation import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderId.PAPAGO import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.CsvSource import org.mockito.ArgumentCaptor import org.mockito.Mockito import org.springframework.scheduling.annotation.Scheduled @@ -14,12 +18,13 @@ import java.util.Optional class TranslationJobWorkerTest { @Test - fun shouldRunEveryTenMinutesByDefault() { + @DisplayName("번역 워커의 기본 실행 간격은 1초다") + fun shouldRunEverySecondByDefault() { val scheduled = TranslationJobWorker::class.java .getDeclaredMethod("runPendingJobs") .getAnnotation(Scheduled::class.java) - assertEquals("\${sodalive.translation-job.fixed-delay-ms:600000}", scheduled.fixedDelayString) + assertEquals("\${sodalive.translation-job.fixed-delay-ms:1000}", scheduled.fixedDelayString) } @Test @@ -210,8 +215,14 @@ class TranslationJobWorkerTest { assertEquals(setOf(1L, 2L, 3L, 4L, 5L), completedResourceIds) } - @Test - fun shouldRetryFailedJobWithBackoff() { + @ParameterizedTest(name = "기존 실패 {0}회 뒤 실패하면 {1}, 대기 {2}분") + @CsvSource("0, PENDING, 1", "1, PENDING, 5", "2, PENDING, 15", "3, PENDING, 15", "4, PENDING, 15", "5, FAILED, 0") + @DisplayName("최초 시도와 재시도 5회의 실패 횟수를 누적하고 여섯 번째 실패에 종료한다") + fun shouldScheduleFiveRetriesBeforeFailingOnSixthAttempt( + failuresBeforeAttempt: Int, + expectedStatus: TranslationJobStatus, + delayMinutes: Long + ) { val jobRepository = Mockito.mock(TranslationJobRepository::class.java) val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java) val provider = failingProvider() @@ -229,17 +240,54 @@ class TranslationJobWorkerTest { ) val job = translationJob() job.id = 200L - val beforeRetryAt = job.nextRetryAt + job.retryCount = failuresBeforeAttempt Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L)) Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job)) - worker.processNextGroup() + val attemptStartedAt = LocalDateTime.now() + assertTrue(worker.processNextGroup()) + val attemptFinishedAt = LocalDateTime.now() - assertEquals(TranslationJobStatus.PENDING, job.status) - assertEquals(1, job.retryCount) + assertEquals(expectedStatus, job.status, "failure #${failuresBeforeAttempt + 1}") + assertEquals(failuresBeforeAttempt + 1, job.retryCount) assertEquals("all translation providers failed", job.lastErrorMessage) - assertTrue(job.nextRetryAt.isAfter(beforeRetryAt)) + if (expectedStatus == TranslationJobStatus.PENDING) { + assertTrue(!job.nextRetryAt.isBefore(attemptStartedAt.plusMinutes(delayMinutes))) + assertTrue(!job.nextRetryAt.isAfter(attemptFinishedAt.plusMinutes(delayMinutes))) + } + } + + @Test + @DisplayName("다섯 번 실패한 작업도 여섯 번째 시도가 성공하면 완료된다") + fun shouldCompleteSixthAttemptAfterFiveFailures() { + val jobRepository = Mockito.mock(TranslationJobRepository::class.java) + val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java) + val materializer = Mockito.mock(TranslationReadModelMaterializer::class.java) + val worker = TranslationJobWorker( + translationJobRepository = jobRepository, + translationMemoryRepository = memoryRepository, + translationCoordinator = TranslationCoordinator( + listOf(successfulProvider()), + LanguageProviderProperties(listOf(PAPAGO), listOf(PAPAGO)), + memoryRepository + ), + materializer = materializer, + transactionManager = TestTransactionManager() + ) + val job = translationJob().also { + it.id = 200L + it.retryCount = 5 + it.lastErrorMessage = "previous failure" + } + Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L)) + Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job)) + + assertTrue(worker.processNextGroup()) + + assertEquals(TranslationJobStatus.COMPLETED, job.status) + assertEquals(5, job.retryCount) + assertNull(job.lastErrorMessage) } private fun translationJob(