fix(i18n): 번역 작업을 1초 주기로 처리하고 실패 작업을 재개한다
This commit is contained in:
+294
@@ -0,0 +1,294 @@
|
||||
package kr.co.vividnext.sodalive.i18n.translation
|
||||
|
||||
import kr.co.vividnext.sodalive.configs.QueryDslConfig
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.DisplayName
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.params.ParameterizedTest
|
||||
import org.junit.jupiter.params.provider.EnumSource
|
||||
import org.junit.jupiter.params.provider.ValueSource
|
||||
import org.mockito.AdditionalAnswers
|
||||
import org.mockito.Mockito
|
||||
import org.springframework.beans.factory.annotation.Autowired
|
||||
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
|
||||
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
|
||||
import org.springframework.context.annotation.Import
|
||||
import org.springframework.data.jpa.repository.Query
|
||||
import org.springframework.transaction.PlatformTransactionManager
|
||||
import org.springframework.transaction.annotation.Propagation
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import org.springframework.transaction.support.TransactionTemplate
|
||||
import java.time.LocalDateTime
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import javax.persistence.EntityManager
|
||||
|
||||
@DataJpaTest(
|
||||
properties = [
|
||||
"spring.cache.type=none",
|
||||
"spring.datasource.url=jdbc:h2:mem:translation-job-retry;MODE=MySQL;NON_KEYWORDS=VALUE;DB_CLOSE_ON_EXIT=FALSE"
|
||||
]
|
||||
)
|
||||
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
|
||||
@Import(QueryDslConfig::class, TranslationJobScheduler::class)
|
||||
class TranslationJobRetryIntegrationTest @Autowired constructor(
|
||||
private val repository: TranslationJobRepository,
|
||||
private val scheduler: TranslationJobScheduler,
|
||||
private val entityManager: EntityManager,
|
||||
private val transactionManager: PlatformTransactionManager
|
||||
) {
|
||||
private val transaction = TransactionTemplate(transactionManager)
|
||||
private val requiresNewTransaction = TransactionTemplate(transactionManager).apply {
|
||||
propagationBehavior = Propagation.REQUIRES_NEW.value()
|
||||
}
|
||||
private val previousRetryAt = LocalDateTime.of(2040, 1, 1, 0, 0)
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints = [3, 6])
|
||||
@DisplayName("기존 실패 횟수가 3회 또는 6회인 FAILED 행을 같은 원문 요청으로 즉시 재개한다")
|
||||
fun shouldResetSameFailedRowForCurrentNormalizedSource(previousFailures: Int) {
|
||||
val original = persistJob(retryCount = previousFailures)
|
||||
val requestStartedAt = LocalDateTime.now().withNano(0)
|
||||
|
||||
schedule()
|
||||
|
||||
val requestFinishedAt = LocalDateTime.now().plusSeconds(1).withNano(0)
|
||||
entityManager.flush()
|
||||
entityManager.clear()
|
||||
val resumed = repository.findAll().single()
|
||||
assertEquals(original.id, resumed.id)
|
||||
assertEquals("제목", resumed.sourceText)
|
||||
assertEquals(SourceTextNormalizer.hash("제목"), resumed.sourceHash)
|
||||
assertEquals(TranslationJobStatus.PENDING, resumed.status)
|
||||
assertEquals(0, resumed.retryCount)
|
||||
assertNull(resumed.lastErrorMessage)
|
||||
assertTrue(!resumed.nextRetryAt.isBefore(requestStartedAt))
|
||||
assertTrue(resumed.nextRetryAt.isBefore(requestFinishedAt))
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@EnumSource(TranslationJobStatus::class, names = ["PENDING", "RUNNING", "COMPLETED"])
|
||||
@DisplayName("반복 요청은 진행 중이거나 완료된 행의 상태와 재시도 예산을 바꾸지 않는다")
|
||||
fun shouldPreserveExistingRowOutsideFailedStatus(status: TranslationJobStatus) {
|
||||
val original = persistJob(status = status, retryCount = 4)
|
||||
|
||||
repeat(2) { schedule() }
|
||||
|
||||
entityManager.flush()
|
||||
entityManager.clear()
|
||||
val preserved = repository.findAll().single()
|
||||
assertEquals(original.id, preserved.id)
|
||||
assertEquals(status, preserved.status)
|
||||
assertEquals(4, preserved.retryCount)
|
||||
assertEquals("previous failure", preserved.lastErrorMessage)
|
||||
assertEquals(previousRetryAt, preserved.nextRetryAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("같은 원문 해시라도 원문 언어가 다른 FAILED 행은 재개하지 않는다")
|
||||
fun shouldNotRestartFailedJobForDifferentSourceLanguage() {
|
||||
val original = persistJob(sourceLanguage = "ja")
|
||||
|
||||
schedule()
|
||||
|
||||
entityManager.flush()
|
||||
entityManager.clear()
|
||||
val preserved = repository.findAll().single()
|
||||
assertEquals(original.id, preserved.id)
|
||||
assertEquals("ja", preserved.sourceLanguage)
|
||||
assertEquals(TranslationJobStatus.FAILED, preserved.status)
|
||||
assertEquals(6, preserved.retryCount)
|
||||
assertEquals("previous failure", preserved.lastErrorMessage)
|
||||
assertEquals(previousRetryAt, preserved.nextRetryAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("현재 원문 요청은 이전 원문의 FAILED 행을 재개하지 않고 현재 해시의 작업을 생성한다")
|
||||
fun shouldLeaveFailedRowForPreviousSourceUntouched() {
|
||||
val previous = persistJob(sourceText = "이전 제목")
|
||||
|
||||
schedule()
|
||||
|
||||
entityManager.flush()
|
||||
entityManager.clear()
|
||||
val jobs = repository.findAll()
|
||||
assertEquals(2, jobs.size)
|
||||
val unchanged = jobs.single { it.id == previous.id }
|
||||
assertEquals(TranslationJobStatus.FAILED, unchanged.status)
|
||||
assertEquals(6, unchanged.retryCount)
|
||||
assertEquals("previous failure", unchanged.lastErrorMessage)
|
||||
assertEquals(previousRetryAt, unchanged.nextRetryAt)
|
||||
val current = jobs.single { it.sourceHash == SourceTextNormalizer.hash("제목") }
|
||||
assertEquals(TranslationJobStatus.PENDING, current.status)
|
||||
assertEquals(0, current.retryCount)
|
||||
assertNull(current.lastErrorMessage)
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("재시도 시각이 지난 FAILED 행도 요청 없이 워커가 자동 선점하지 않는다")
|
||||
fun shouldNotPollFailedRowWithoutNewRequest() {
|
||||
val original = persistJob().apply { nextRetryAt = LocalDateTime.of(2000, 1, 1, 0, 0) }
|
||||
entityManager.flush()
|
||||
val claimSql = TranslationJobRepository::class.java
|
||||
.getMethod("findNextPendingGroupJobIdsForUpdate", LocalDateTime::class.java)
|
||||
.getAnnotation(Query::class.java).value
|
||||
.replace("skip locked", "") // H2에서는 잠금 옵션만 제외하고 실제 운영 쿼리의 선점 조건을 실행한다.
|
||||
val claimRepository = Mockito.mock(
|
||||
TranslationJobRepository::class.java,
|
||||
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
|
||||
)
|
||||
Mockito.doAnswer { call ->
|
||||
entityManager.createNativeQuery(claimSql)
|
||||
.setParameter("now", call.getArgument<LocalDateTime>(0))
|
||||
.resultList.map { (it as Number).toLong() }
|
||||
}.`when`(claimRepository).findNextPendingGroupJobIdsForUpdate(
|
||||
Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now()
|
||||
)
|
||||
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
|
||||
val provider = object : TranslationProvider {
|
||||
override val providerId = LanguageProviderId.PAPAGO
|
||||
override val providerVersion = "test"
|
||||
override fun translate(request: TranslateRequest): TranslateResult =
|
||||
throw IllegalStateException("unexpected translation")
|
||||
}
|
||||
val worker = TranslationJobWorker(
|
||||
claimRepository,
|
||||
memoryRepository,
|
||||
TranslationCoordinator(
|
||||
listOf(provider),
|
||||
LanguageProviderProperties(listOf(LanguageProviderId.PAPAGO), listOf(LanguageProviderId.PAPAGO)),
|
||||
memoryRepository
|
||||
),
|
||||
Mockito.mock(TranslationReadModelMaterializer::class.java),
|
||||
transactionManager
|
||||
)
|
||||
|
||||
assertFalse(worker.processNextGroup())
|
||||
|
||||
entityManager.flush()
|
||||
entityManager.clear()
|
||||
val untouched = repository.findAll().single()
|
||||
assertEquals(original.id, untouched.id)
|
||||
assertEquals(TranslationJobStatus.FAILED, untouched.status)
|
||||
assertEquals(6, untouched.retryCount)
|
||||
assertEquals("previous failure", untouched.lastErrorMessage)
|
||||
assertEquals(LocalDateTime.of(2000, 1, 1, 0, 0), untouched.nextRetryAt)
|
||||
}
|
||||
|
||||
@Test
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
@DisplayName("같은 FAILED 행의 동시 재개 요청은 중복 행 없이 하나의 새 주기만 연다")
|
||||
fun shouldResumeSameFailedRowFromConcurrentRequests() {
|
||||
val original = persistJob()
|
||||
val originalId = requireNotNull(original.id)
|
||||
val executor = Executors.newFixedThreadPool(2)
|
||||
val ready = CountDownLatch(2)
|
||||
val start = CountDownLatch(1)
|
||||
try {
|
||||
val requests = (1..2).map {
|
||||
executor.submit {
|
||||
ready.countDown()
|
||||
check(start.await(5, TimeUnit.SECONDS)) { "concurrent request start timed out" }
|
||||
schedule()
|
||||
}
|
||||
}
|
||||
assertTrue(ready.await(5, TimeUnit.SECONDS))
|
||||
start.countDown()
|
||||
requests.forEach { it.get(5, TimeUnit.SECONDS) }
|
||||
|
||||
val resumed = repository.findAll().single()
|
||||
assertEquals(originalId, resumed.id)
|
||||
assertEquals(TranslationJobStatus.PENDING, resumed.status)
|
||||
assertEquals(0, resumed.retryCount)
|
||||
assertNull(resumed.lastErrorMessage)
|
||||
} finally {
|
||||
start.countDown()
|
||||
executor.shutdownNow()
|
||||
check(executor.awaitTermination(5, TimeUnit.SECONDS)) { "concurrent requests did not stop" }
|
||||
repository.deleteById(originalId)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
@DisplayName("FAILED 조회 뒤 다른 트랜잭션이 RUNNING으로 선점하면 오래된 재개 요청은 예산을 초기화하지 않는다")
|
||||
fun shouldPreserveClaimCommittedAfterFailedSnapshotWasRead() {
|
||||
val originalId = requireNotNull(persistJob().id)
|
||||
val interleavingRepository = Mockito.mock(
|
||||
TranslationJobRepository::class.java,
|
||||
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
|
||||
)
|
||||
Mockito.doAnswer {
|
||||
val snapshot = repository.findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
|
||||
LanguageTranslationTargetType.CONTENT,
|
||||
10L,
|
||||
"title",
|
||||
"en",
|
||||
SourceTextNormalizer.hash("제목")
|
||||
)
|
||||
requiresNewTransaction.executeWithoutResult {
|
||||
val claimed = repository.findById(originalId).orElseThrow()
|
||||
claimed.status = TranslationJobStatus.RUNNING
|
||||
claimed.retryCount = 4
|
||||
repository.saveAndFlush(claimed)
|
||||
}
|
||||
snapshot
|
||||
}.`when`(interleavingRepository).findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
|
||||
LanguageTranslationTargetType.CONTENT,
|
||||
10L,
|
||||
"title",
|
||||
"en",
|
||||
SourceTextNormalizer.hash("제목")
|
||||
)
|
||||
try {
|
||||
transaction.executeWithoutResult {
|
||||
schedule(TranslationJobScheduler(interleavingRepository))
|
||||
}
|
||||
|
||||
val claimed = repository.findAll().single()
|
||||
assertEquals(originalId, claimed.id)
|
||||
assertEquals(TranslationJobStatus.RUNNING, claimed.status)
|
||||
assertEquals(4, claimed.retryCount)
|
||||
assertEquals("previous failure", claimed.lastErrorMessage)
|
||||
assertEquals(previousRetryAt, claimed.nextRetryAt)
|
||||
} finally {
|
||||
repository.deleteById(originalId)
|
||||
}
|
||||
}
|
||||
|
||||
private fun schedule(scheduler: TranslationJobScheduler = this.scheduler) {
|
||||
scheduler.scheduleMissingTranslation(
|
||||
resourceType = LanguageTranslationTargetType.CONTENT,
|
||||
resourceId = 10L,
|
||||
fieldKey = "title",
|
||||
sourceText = " 제목\n",
|
||||
sourceLanguage = "ko",
|
||||
targetLanguage = "EN"
|
||||
)
|
||||
}
|
||||
|
||||
private fun persistJob(
|
||||
status: TranslationJobStatus = TranslationJobStatus.FAILED,
|
||||
sourceText: String = "제목",
|
||||
retryCount: Int = 6,
|
||||
sourceLanguage: String = "ko"
|
||||
): TranslationJob = repository.saveAndFlush(
|
||||
TranslationJob(
|
||||
resourceType = LanguageTranslationTargetType.CONTENT,
|
||||
resourceId = 10L,
|
||||
fieldKey = "title",
|
||||
sourceHash = SourceTextNormalizer.hash(sourceText),
|
||||
sourceText = sourceText,
|
||||
sourceLanguage = sourceLanguage,
|
||||
targetLanguage = "en",
|
||||
status = status,
|
||||
retryCount = retryCount,
|
||||
lastErrorMessage = "previous failure",
|
||||
nextRetryAt = previousRetryAt
|
||||
)
|
||||
)
|
||||
}
|
||||
+57
-9
@@ -2,8 +2,12 @@ package kr.co.vividnext.sodalive.i18n.translation
|
||||
|
||||
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderId.PAPAGO
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertNull
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.DisplayName
|
||||
import org.junit.jupiter.api.Test
|
||||
import org.junit.jupiter.params.ParameterizedTest
|
||||
import org.junit.jupiter.params.provider.CsvSource
|
||||
import org.mockito.ArgumentCaptor
|
||||
import org.mockito.Mockito
|
||||
import org.springframework.scheduling.annotation.Scheduled
|
||||
@@ -14,12 +18,13 @@ import java.util.Optional
|
||||
|
||||
class TranslationJobWorkerTest {
|
||||
@Test
|
||||
fun shouldRunEveryTenMinutesByDefault() {
|
||||
@DisplayName("번역 워커의 기본 실행 간격은 1초다")
|
||||
fun shouldRunEverySecondByDefault() {
|
||||
val scheduled = TranslationJobWorker::class.java
|
||||
.getDeclaredMethod("runPendingJobs")
|
||||
.getAnnotation(Scheduled::class.java)
|
||||
|
||||
assertEquals("\${sodalive.translation-job.fixed-delay-ms:600000}", scheduled.fixedDelayString)
|
||||
assertEquals("\${sodalive.translation-job.fixed-delay-ms:1000}", scheduled.fixedDelayString)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -210,8 +215,14 @@ class TranslationJobWorkerTest {
|
||||
assertEquals(setOf(1L, 2L, 3L, 4L, 5L), completedResourceIds)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun shouldRetryFailedJobWithBackoff() {
|
||||
@ParameterizedTest(name = "기존 실패 {0}회 뒤 실패하면 {1}, 대기 {2}분")
|
||||
@CsvSource("0, PENDING, 1", "1, PENDING, 5", "2, PENDING, 15", "3, PENDING, 15", "4, PENDING, 15", "5, FAILED, 0")
|
||||
@DisplayName("최초 시도와 재시도 5회의 실패 횟수를 누적하고 여섯 번째 실패에 종료한다")
|
||||
fun shouldScheduleFiveRetriesBeforeFailingOnSixthAttempt(
|
||||
failuresBeforeAttempt: Int,
|
||||
expectedStatus: TranslationJobStatus,
|
||||
delayMinutes: Long
|
||||
) {
|
||||
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
|
||||
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
|
||||
val provider = failingProvider()
|
||||
@@ -229,17 +240,54 @@ class TranslationJobWorkerTest {
|
||||
)
|
||||
val job = translationJob()
|
||||
job.id = 200L
|
||||
val beforeRetryAt = job.nextRetryAt
|
||||
job.retryCount = failuresBeforeAttempt
|
||||
|
||||
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
|
||||
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
|
||||
|
||||
worker.processNextGroup()
|
||||
val attemptStartedAt = LocalDateTime.now()
|
||||
assertTrue(worker.processNextGroup())
|
||||
val attemptFinishedAt = LocalDateTime.now()
|
||||
|
||||
assertEquals(TranslationJobStatus.PENDING, job.status)
|
||||
assertEquals(1, job.retryCount)
|
||||
assertEquals(expectedStatus, job.status, "failure #${failuresBeforeAttempt + 1}")
|
||||
assertEquals(failuresBeforeAttempt + 1, job.retryCount)
|
||||
assertEquals("all translation providers failed", job.lastErrorMessage)
|
||||
assertTrue(job.nextRetryAt.isAfter(beforeRetryAt))
|
||||
if (expectedStatus == TranslationJobStatus.PENDING) {
|
||||
assertTrue(!job.nextRetryAt.isBefore(attemptStartedAt.plusMinutes(delayMinutes)))
|
||||
assertTrue(!job.nextRetryAt.isAfter(attemptFinishedAt.plusMinutes(delayMinutes)))
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("다섯 번 실패한 작업도 여섯 번째 시도가 성공하면 완료된다")
|
||||
fun shouldCompleteSixthAttemptAfterFiveFailures() {
|
||||
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
|
||||
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
|
||||
val materializer = Mockito.mock(TranslationReadModelMaterializer::class.java)
|
||||
val worker = TranslationJobWorker(
|
||||
translationJobRepository = jobRepository,
|
||||
translationMemoryRepository = memoryRepository,
|
||||
translationCoordinator = TranslationCoordinator(
|
||||
listOf(successfulProvider()),
|
||||
LanguageProviderProperties(listOf(PAPAGO), listOf(PAPAGO)),
|
||||
memoryRepository
|
||||
),
|
||||
materializer = materializer,
|
||||
transactionManager = TestTransactionManager()
|
||||
)
|
||||
val job = translationJob().also {
|
||||
it.id = 200L
|
||||
it.retryCount = 5
|
||||
it.lastErrorMessage = "previous failure"
|
||||
}
|
||||
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
|
||||
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
|
||||
|
||||
assertTrue(worker.processNextGroup())
|
||||
|
||||
assertEquals(TranslationJobStatus.COMPLETED, job.status)
|
||||
assertEquals(5, job.retryCount)
|
||||
assertNull(job.lastErrorMessage)
|
||||
}
|
||||
|
||||
private fun translationJob(
|
||||
|
||||
Reference in New Issue
Block a user