fix(i18n): 번역 작업을 1초 주기로 처리하고 실패 작업을 재개한다
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
package kr.co.vividnext.sodalive.i18n.translation
|
package kr.co.vividnext.sodalive.i18n.translation
|
||||||
|
|
||||||
import org.springframework.data.jpa.repository.JpaRepository
|
import org.springframework.data.jpa.repository.JpaRepository
|
||||||
|
import org.springframework.data.jpa.repository.Modifying
|
||||||
import org.springframework.data.jpa.repository.Query
|
import org.springframework.data.jpa.repository.Query
|
||||||
import org.springframework.data.repository.query.Param
|
import org.springframework.data.repository.query.Param
|
||||||
import java.time.LocalDateTime
|
import java.time.LocalDateTime
|
||||||
@@ -14,6 +15,27 @@ interface TranslationJobRepository : JpaRepository<TranslationJob, Long> {
|
|||||||
sourceHash: String
|
sourceHash: String
|
||||||
): TranslationJob?
|
): TranslationJob?
|
||||||
|
|
||||||
|
fun findByResourceTypeAndResourceIdInAndTargetLanguageAndStatus(
|
||||||
|
resourceType: LanguageTranslationTargetType,
|
||||||
|
resourceIds: Collection<Long>,
|
||||||
|
targetLanguage: String,
|
||||||
|
status: TranslationJobStatus
|
||||||
|
): List<TranslationJob>
|
||||||
|
|
||||||
|
@Modifying
|
||||||
|
@Query(
|
||||||
|
"""
|
||||||
|
update TranslationJob j
|
||||||
|
set j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.PENDING,
|
||||||
|
j.retryCount = 0,
|
||||||
|
j.lastErrorMessage = null,
|
||||||
|
j.nextRetryAt = :now
|
||||||
|
where j.id = :jobId
|
||||||
|
and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.FAILED
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
fun restartFailedJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int
|
||||||
|
|
||||||
@Query(
|
@Query(
|
||||||
"""
|
"""
|
||||||
select j from TranslationJob j
|
select j from TranslationJob j
|
||||||
|
|||||||
+6
-1
@@ -34,7 +34,12 @@ class TranslationJobScheduler(
|
|||||||
targetLanguage = normalizedTargetLanguage,
|
targetLanguage = normalizedTargetLanguage,
|
||||||
sourceHash = sourceHash
|
sourceHash = sourceHash
|
||||||
)
|
)
|
||||||
if (existingJob != null) return
|
if (existingJob != null) {
|
||||||
|
if (existingJob.status == TranslationJobStatus.FAILED && existingJob.sourceLanguage == normalizedSourceLanguage) {
|
||||||
|
translationJobRepository.restartFailedJob(existingJob.id!!, LocalDateTime.now())
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
translationJobRepository.save(
|
translationJobRepository.save(
|
||||||
TranslationJob(
|
TranslationJob(
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ class TranslationJobWorker(
|
|||||||
private val log = LoggerFactory.getLogger(javaClass)
|
private val log = LoggerFactory.getLogger(javaClass)
|
||||||
private val transactionTemplate = TransactionTemplate(transactionManager)
|
private val transactionTemplate = TransactionTemplate(transactionManager)
|
||||||
|
|
||||||
@Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:600000}")
|
@Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:1000}")
|
||||||
fun runPendingJobs() {
|
fun runPendingJobs() {
|
||||||
repeat(MAX_GROUPS_PER_TICK) {
|
repeat(MAX_GROUPS_PER_TICK) {
|
||||||
if (!processNextGroup()) return
|
if (!processNextGroup()) return
|
||||||
@@ -121,7 +121,7 @@ class TranslationJobWorker(
|
|||||||
val job = translationJobRepository.findById(jobId).orElse(null) ?: return@executeWithoutResult
|
val job = translationJobRepository.findById(jobId).orElse(null) ?: return@executeWithoutResult
|
||||||
job.retryCount += 1
|
job.retryCount += 1
|
||||||
job.lastErrorMessage = ex.message?.take(MAX_ERROR_LENGTH)
|
job.lastErrorMessage = ex.message?.take(MAX_ERROR_LENGTH)
|
||||||
if (job.retryCount >= MAX_RETRY_COUNT) {
|
if (job.retryCount >= MAX_ATTEMPTS) {
|
||||||
job.status = TranslationJobStatus.FAILED
|
job.status = TranslationJobStatus.FAILED
|
||||||
} else {
|
} else {
|
||||||
job.status = TranslationJobStatus.PENDING
|
job.status = TranslationJobStatus.PENDING
|
||||||
@@ -142,6 +142,6 @@ class TranslationJobWorker(
|
|||||||
companion object {
|
companion object {
|
||||||
private const val MAX_GROUPS_PER_TICK = 5
|
private const val MAX_GROUPS_PER_TICK = 5
|
||||||
private const val MAX_ERROR_LENGTH = 1000
|
private const val MAX_ERROR_LENGTH = 1000
|
||||||
private const val MAX_RETRY_COUNT = 3
|
private const val MAX_ATTEMPTS = 6
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+294
@@ -0,0 +1,294 @@
|
|||||||
|
package kr.co.vividnext.sodalive.i18n.translation
|
||||||
|
|
||||||
|
import kr.co.vividnext.sodalive.configs.QueryDslConfig
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Assertions.assertFalse
|
||||||
|
import org.junit.jupiter.api.Assertions.assertNull
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.DisplayName
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import org.junit.jupiter.params.ParameterizedTest
|
||||||
|
import org.junit.jupiter.params.provider.EnumSource
|
||||||
|
import org.junit.jupiter.params.provider.ValueSource
|
||||||
|
import org.mockito.AdditionalAnswers
|
||||||
|
import org.mockito.Mockito
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired
|
||||||
|
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
|
||||||
|
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
|
||||||
|
import org.springframework.context.annotation.Import
|
||||||
|
import org.springframework.data.jpa.repository.Query
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager
|
||||||
|
import org.springframework.transaction.annotation.Propagation
|
||||||
|
import org.springframework.transaction.annotation.Transactional
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate
|
||||||
|
import java.time.LocalDateTime
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
import javax.persistence.EntityManager
|
||||||
|
|
||||||
|
@DataJpaTest(
|
||||||
|
properties = [
|
||||||
|
"spring.cache.type=none",
|
||||||
|
"spring.datasource.url=jdbc:h2:mem:translation-job-retry;MODE=MySQL;NON_KEYWORDS=VALUE;DB_CLOSE_ON_EXIT=FALSE"
|
||||||
|
]
|
||||||
|
)
|
||||||
|
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
|
||||||
|
@Import(QueryDslConfig::class, TranslationJobScheduler::class)
|
||||||
|
class TranslationJobRetryIntegrationTest @Autowired constructor(
|
||||||
|
private val repository: TranslationJobRepository,
|
||||||
|
private val scheduler: TranslationJobScheduler,
|
||||||
|
private val entityManager: EntityManager,
|
||||||
|
private val transactionManager: PlatformTransactionManager
|
||||||
|
) {
|
||||||
|
private val transaction = TransactionTemplate(transactionManager)
|
||||||
|
private val requiresNewTransaction = TransactionTemplate(transactionManager).apply {
|
||||||
|
propagationBehavior = Propagation.REQUIRES_NEW.value()
|
||||||
|
}
|
||||||
|
private val previousRetryAt = LocalDateTime.of(2040, 1, 1, 0, 0)
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@ValueSource(ints = [3, 6])
|
||||||
|
@DisplayName("기존 실패 횟수가 3회 또는 6회인 FAILED 행을 같은 원문 요청으로 즉시 재개한다")
|
||||||
|
fun shouldResetSameFailedRowForCurrentNormalizedSource(previousFailures: Int) {
|
||||||
|
val original = persistJob(retryCount = previousFailures)
|
||||||
|
val requestStartedAt = LocalDateTime.now().withNano(0)
|
||||||
|
|
||||||
|
schedule()
|
||||||
|
|
||||||
|
val requestFinishedAt = LocalDateTime.now().plusSeconds(1).withNano(0)
|
||||||
|
entityManager.flush()
|
||||||
|
entityManager.clear()
|
||||||
|
val resumed = repository.findAll().single()
|
||||||
|
assertEquals(original.id, resumed.id)
|
||||||
|
assertEquals("제목", resumed.sourceText)
|
||||||
|
assertEquals(SourceTextNormalizer.hash("제목"), resumed.sourceHash)
|
||||||
|
assertEquals(TranslationJobStatus.PENDING, resumed.status)
|
||||||
|
assertEquals(0, resumed.retryCount)
|
||||||
|
assertNull(resumed.lastErrorMessage)
|
||||||
|
assertTrue(!resumed.nextRetryAt.isBefore(requestStartedAt))
|
||||||
|
assertTrue(resumed.nextRetryAt.isBefore(requestFinishedAt))
|
||||||
|
}
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@EnumSource(TranslationJobStatus::class, names = ["PENDING", "RUNNING", "COMPLETED"])
|
||||||
|
@DisplayName("반복 요청은 진행 중이거나 완료된 행의 상태와 재시도 예산을 바꾸지 않는다")
|
||||||
|
fun shouldPreserveExistingRowOutsideFailedStatus(status: TranslationJobStatus) {
|
||||||
|
val original = persistJob(status = status, retryCount = 4)
|
||||||
|
|
||||||
|
repeat(2) { schedule() }
|
||||||
|
|
||||||
|
entityManager.flush()
|
||||||
|
entityManager.clear()
|
||||||
|
val preserved = repository.findAll().single()
|
||||||
|
assertEquals(original.id, preserved.id)
|
||||||
|
assertEquals(status, preserved.status)
|
||||||
|
assertEquals(4, preserved.retryCount)
|
||||||
|
assertEquals("previous failure", preserved.lastErrorMessage)
|
||||||
|
assertEquals(previousRetryAt, preserved.nextRetryAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("같은 원문 해시라도 원문 언어가 다른 FAILED 행은 재개하지 않는다")
|
||||||
|
fun shouldNotRestartFailedJobForDifferentSourceLanguage() {
|
||||||
|
val original = persistJob(sourceLanguage = "ja")
|
||||||
|
|
||||||
|
schedule()
|
||||||
|
|
||||||
|
entityManager.flush()
|
||||||
|
entityManager.clear()
|
||||||
|
val preserved = repository.findAll().single()
|
||||||
|
assertEquals(original.id, preserved.id)
|
||||||
|
assertEquals("ja", preserved.sourceLanguage)
|
||||||
|
assertEquals(TranslationJobStatus.FAILED, preserved.status)
|
||||||
|
assertEquals(6, preserved.retryCount)
|
||||||
|
assertEquals("previous failure", preserved.lastErrorMessage)
|
||||||
|
assertEquals(previousRetryAt, preserved.nextRetryAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("현재 원문 요청은 이전 원문의 FAILED 행을 재개하지 않고 현재 해시의 작업을 생성한다")
|
||||||
|
fun shouldLeaveFailedRowForPreviousSourceUntouched() {
|
||||||
|
val previous = persistJob(sourceText = "이전 제목")
|
||||||
|
|
||||||
|
schedule()
|
||||||
|
|
||||||
|
entityManager.flush()
|
||||||
|
entityManager.clear()
|
||||||
|
val jobs = repository.findAll()
|
||||||
|
assertEquals(2, jobs.size)
|
||||||
|
val unchanged = jobs.single { it.id == previous.id }
|
||||||
|
assertEquals(TranslationJobStatus.FAILED, unchanged.status)
|
||||||
|
assertEquals(6, unchanged.retryCount)
|
||||||
|
assertEquals("previous failure", unchanged.lastErrorMessage)
|
||||||
|
assertEquals(previousRetryAt, unchanged.nextRetryAt)
|
||||||
|
val current = jobs.single { it.sourceHash == SourceTextNormalizer.hash("제목") }
|
||||||
|
assertEquals(TranslationJobStatus.PENDING, current.status)
|
||||||
|
assertEquals(0, current.retryCount)
|
||||||
|
assertNull(current.lastErrorMessage)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("재시도 시각이 지난 FAILED 행도 요청 없이 워커가 자동 선점하지 않는다")
|
||||||
|
fun shouldNotPollFailedRowWithoutNewRequest() {
|
||||||
|
val original = persistJob().apply { nextRetryAt = LocalDateTime.of(2000, 1, 1, 0, 0) }
|
||||||
|
entityManager.flush()
|
||||||
|
val claimSql = TranslationJobRepository::class.java
|
||||||
|
.getMethod("findNextPendingGroupJobIdsForUpdate", LocalDateTime::class.java)
|
||||||
|
.getAnnotation(Query::class.java).value
|
||||||
|
.replace("skip locked", "") // H2에서는 잠금 옵션만 제외하고 실제 운영 쿼리의 선점 조건을 실행한다.
|
||||||
|
val claimRepository = Mockito.mock(
|
||||||
|
TranslationJobRepository::class.java,
|
||||||
|
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
|
||||||
|
)
|
||||||
|
Mockito.doAnswer { call ->
|
||||||
|
entityManager.createNativeQuery(claimSql)
|
||||||
|
.setParameter("now", call.getArgument<LocalDateTime>(0))
|
||||||
|
.resultList.map { (it as Number).toLong() }
|
||||||
|
}.`when`(claimRepository).findNextPendingGroupJobIdsForUpdate(
|
||||||
|
Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now()
|
||||||
|
)
|
||||||
|
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
|
||||||
|
val provider = object : TranslationProvider {
|
||||||
|
override val providerId = LanguageProviderId.PAPAGO
|
||||||
|
override val providerVersion = "test"
|
||||||
|
override fun translate(request: TranslateRequest): TranslateResult =
|
||||||
|
throw IllegalStateException("unexpected translation")
|
||||||
|
}
|
||||||
|
val worker = TranslationJobWorker(
|
||||||
|
claimRepository,
|
||||||
|
memoryRepository,
|
||||||
|
TranslationCoordinator(
|
||||||
|
listOf(provider),
|
||||||
|
LanguageProviderProperties(listOf(LanguageProviderId.PAPAGO), listOf(LanguageProviderId.PAPAGO)),
|
||||||
|
memoryRepository
|
||||||
|
),
|
||||||
|
Mockito.mock(TranslationReadModelMaterializer::class.java),
|
||||||
|
transactionManager
|
||||||
|
)
|
||||||
|
|
||||||
|
assertFalse(worker.processNextGroup())
|
||||||
|
|
||||||
|
entityManager.flush()
|
||||||
|
entityManager.clear()
|
||||||
|
val untouched = repository.findAll().single()
|
||||||
|
assertEquals(original.id, untouched.id)
|
||||||
|
assertEquals(TranslationJobStatus.FAILED, untouched.status)
|
||||||
|
assertEquals(6, untouched.retryCount)
|
||||||
|
assertEquals("previous failure", untouched.lastErrorMessage)
|
||||||
|
assertEquals(LocalDateTime.of(2000, 1, 1, 0, 0), untouched.nextRetryAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||||
|
@DisplayName("같은 FAILED 행의 동시 재개 요청은 중복 행 없이 하나의 새 주기만 연다")
|
||||||
|
fun shouldResumeSameFailedRowFromConcurrentRequests() {
|
||||||
|
val original = persistJob()
|
||||||
|
val originalId = requireNotNull(original.id)
|
||||||
|
val executor = Executors.newFixedThreadPool(2)
|
||||||
|
val ready = CountDownLatch(2)
|
||||||
|
val start = CountDownLatch(1)
|
||||||
|
try {
|
||||||
|
val requests = (1..2).map {
|
||||||
|
executor.submit {
|
||||||
|
ready.countDown()
|
||||||
|
check(start.await(5, TimeUnit.SECONDS)) { "concurrent request start timed out" }
|
||||||
|
schedule()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertTrue(ready.await(5, TimeUnit.SECONDS))
|
||||||
|
start.countDown()
|
||||||
|
requests.forEach { it.get(5, TimeUnit.SECONDS) }
|
||||||
|
|
||||||
|
val resumed = repository.findAll().single()
|
||||||
|
assertEquals(originalId, resumed.id)
|
||||||
|
assertEquals(TranslationJobStatus.PENDING, resumed.status)
|
||||||
|
assertEquals(0, resumed.retryCount)
|
||||||
|
assertNull(resumed.lastErrorMessage)
|
||||||
|
} finally {
|
||||||
|
start.countDown()
|
||||||
|
executor.shutdownNow()
|
||||||
|
check(executor.awaitTermination(5, TimeUnit.SECONDS)) { "concurrent requests did not stop" }
|
||||||
|
repository.deleteById(originalId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||||
|
@DisplayName("FAILED 조회 뒤 다른 트랜잭션이 RUNNING으로 선점하면 오래된 재개 요청은 예산을 초기화하지 않는다")
|
||||||
|
fun shouldPreserveClaimCommittedAfterFailedSnapshotWasRead() {
|
||||||
|
val originalId = requireNotNull(persistJob().id)
|
||||||
|
val interleavingRepository = Mockito.mock(
|
||||||
|
TranslationJobRepository::class.java,
|
||||||
|
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
|
||||||
|
)
|
||||||
|
Mockito.doAnswer {
|
||||||
|
val snapshot = repository.findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
|
||||||
|
LanguageTranslationTargetType.CONTENT,
|
||||||
|
10L,
|
||||||
|
"title",
|
||||||
|
"en",
|
||||||
|
SourceTextNormalizer.hash("제목")
|
||||||
|
)
|
||||||
|
requiresNewTransaction.executeWithoutResult {
|
||||||
|
val claimed = repository.findById(originalId).orElseThrow()
|
||||||
|
claimed.status = TranslationJobStatus.RUNNING
|
||||||
|
claimed.retryCount = 4
|
||||||
|
repository.saveAndFlush(claimed)
|
||||||
|
}
|
||||||
|
snapshot
|
||||||
|
}.`when`(interleavingRepository).findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
|
||||||
|
LanguageTranslationTargetType.CONTENT,
|
||||||
|
10L,
|
||||||
|
"title",
|
||||||
|
"en",
|
||||||
|
SourceTextNormalizer.hash("제목")
|
||||||
|
)
|
||||||
|
try {
|
||||||
|
transaction.executeWithoutResult {
|
||||||
|
schedule(TranslationJobScheduler(interleavingRepository))
|
||||||
|
}
|
||||||
|
|
||||||
|
val claimed = repository.findAll().single()
|
||||||
|
assertEquals(originalId, claimed.id)
|
||||||
|
assertEquals(TranslationJobStatus.RUNNING, claimed.status)
|
||||||
|
assertEquals(4, claimed.retryCount)
|
||||||
|
assertEquals("previous failure", claimed.lastErrorMessage)
|
||||||
|
assertEquals(previousRetryAt, claimed.nextRetryAt)
|
||||||
|
} finally {
|
||||||
|
repository.deleteById(originalId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun schedule(scheduler: TranslationJobScheduler = this.scheduler) {
|
||||||
|
scheduler.scheduleMissingTranslation(
|
||||||
|
resourceType = LanguageTranslationTargetType.CONTENT,
|
||||||
|
resourceId = 10L,
|
||||||
|
fieldKey = "title",
|
||||||
|
sourceText = " 제목\n",
|
||||||
|
sourceLanguage = "ko",
|
||||||
|
targetLanguage = "EN"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun persistJob(
|
||||||
|
status: TranslationJobStatus = TranslationJobStatus.FAILED,
|
||||||
|
sourceText: String = "제목",
|
||||||
|
retryCount: Int = 6,
|
||||||
|
sourceLanguage: String = "ko"
|
||||||
|
): TranslationJob = repository.saveAndFlush(
|
||||||
|
TranslationJob(
|
||||||
|
resourceType = LanguageTranslationTargetType.CONTENT,
|
||||||
|
resourceId = 10L,
|
||||||
|
fieldKey = "title",
|
||||||
|
sourceHash = SourceTextNormalizer.hash(sourceText),
|
||||||
|
sourceText = sourceText,
|
||||||
|
sourceLanguage = sourceLanguage,
|
||||||
|
targetLanguage = "en",
|
||||||
|
status = status,
|
||||||
|
retryCount = retryCount,
|
||||||
|
lastErrorMessage = "previous failure",
|
||||||
|
nextRetryAt = previousRetryAt
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
+57
-9
@@ -2,8 +2,12 @@ package kr.co.vividnext.sodalive.i18n.translation
|
|||||||
|
|
||||||
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderId.PAPAGO
|
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderId.PAPAGO
|
||||||
import org.junit.jupiter.api.Assertions.assertEquals
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Assertions.assertNull
|
||||||
import org.junit.jupiter.api.Assertions.assertTrue
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.DisplayName
|
||||||
import org.junit.jupiter.api.Test
|
import org.junit.jupiter.api.Test
|
||||||
|
import org.junit.jupiter.params.ParameterizedTest
|
||||||
|
import org.junit.jupiter.params.provider.CsvSource
|
||||||
import org.mockito.ArgumentCaptor
|
import org.mockito.ArgumentCaptor
|
||||||
import org.mockito.Mockito
|
import org.mockito.Mockito
|
||||||
import org.springframework.scheduling.annotation.Scheduled
|
import org.springframework.scheduling.annotation.Scheduled
|
||||||
@@ -14,12 +18,13 @@ import java.util.Optional
|
|||||||
|
|
||||||
class TranslationJobWorkerTest {
|
class TranslationJobWorkerTest {
|
||||||
@Test
|
@Test
|
||||||
fun shouldRunEveryTenMinutesByDefault() {
|
@DisplayName("번역 워커의 기본 실행 간격은 1초다")
|
||||||
|
fun shouldRunEverySecondByDefault() {
|
||||||
val scheduled = TranslationJobWorker::class.java
|
val scheduled = TranslationJobWorker::class.java
|
||||||
.getDeclaredMethod("runPendingJobs")
|
.getDeclaredMethod("runPendingJobs")
|
||||||
.getAnnotation(Scheduled::class.java)
|
.getAnnotation(Scheduled::class.java)
|
||||||
|
|
||||||
assertEquals("\${sodalive.translation-job.fixed-delay-ms:600000}", scheduled.fixedDelayString)
|
assertEquals("\${sodalive.translation-job.fixed-delay-ms:1000}", scheduled.fixedDelayString)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -210,8 +215,14 @@ class TranslationJobWorkerTest {
|
|||||||
assertEquals(setOf(1L, 2L, 3L, 4L, 5L), completedResourceIds)
|
assertEquals(setOf(1L, 2L, 3L, 4L, 5L), completedResourceIds)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@ParameterizedTest(name = "기존 실패 {0}회 뒤 실패하면 {1}, 대기 {2}분")
|
||||||
fun shouldRetryFailedJobWithBackoff() {
|
@CsvSource("0, PENDING, 1", "1, PENDING, 5", "2, PENDING, 15", "3, PENDING, 15", "4, PENDING, 15", "5, FAILED, 0")
|
||||||
|
@DisplayName("최초 시도와 재시도 5회의 실패 횟수를 누적하고 여섯 번째 실패에 종료한다")
|
||||||
|
fun shouldScheduleFiveRetriesBeforeFailingOnSixthAttempt(
|
||||||
|
failuresBeforeAttempt: Int,
|
||||||
|
expectedStatus: TranslationJobStatus,
|
||||||
|
delayMinutes: Long
|
||||||
|
) {
|
||||||
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
|
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
|
||||||
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
|
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
|
||||||
val provider = failingProvider()
|
val provider = failingProvider()
|
||||||
@@ -229,17 +240,54 @@ class TranslationJobWorkerTest {
|
|||||||
)
|
)
|
||||||
val job = translationJob()
|
val job = translationJob()
|
||||||
job.id = 200L
|
job.id = 200L
|
||||||
val beforeRetryAt = job.nextRetryAt
|
job.retryCount = failuresBeforeAttempt
|
||||||
|
|
||||||
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
|
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
|
||||||
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
|
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
|
||||||
|
|
||||||
worker.processNextGroup()
|
val attemptStartedAt = LocalDateTime.now()
|
||||||
|
assertTrue(worker.processNextGroup())
|
||||||
|
val attemptFinishedAt = LocalDateTime.now()
|
||||||
|
|
||||||
assertEquals(TranslationJobStatus.PENDING, job.status)
|
assertEquals(expectedStatus, job.status, "failure #${failuresBeforeAttempt + 1}")
|
||||||
assertEquals(1, job.retryCount)
|
assertEquals(failuresBeforeAttempt + 1, job.retryCount)
|
||||||
assertEquals("all translation providers failed", job.lastErrorMessage)
|
assertEquals("all translation providers failed", job.lastErrorMessage)
|
||||||
assertTrue(job.nextRetryAt.isAfter(beforeRetryAt))
|
if (expectedStatus == TranslationJobStatus.PENDING) {
|
||||||
|
assertTrue(!job.nextRetryAt.isBefore(attemptStartedAt.plusMinutes(delayMinutes)))
|
||||||
|
assertTrue(!job.nextRetryAt.isAfter(attemptFinishedAt.plusMinutes(delayMinutes)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("다섯 번 실패한 작업도 여섯 번째 시도가 성공하면 완료된다")
|
||||||
|
fun shouldCompleteSixthAttemptAfterFiveFailures() {
|
||||||
|
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
|
||||||
|
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
|
||||||
|
val materializer = Mockito.mock(TranslationReadModelMaterializer::class.java)
|
||||||
|
val worker = TranslationJobWorker(
|
||||||
|
translationJobRepository = jobRepository,
|
||||||
|
translationMemoryRepository = memoryRepository,
|
||||||
|
translationCoordinator = TranslationCoordinator(
|
||||||
|
listOf(successfulProvider()),
|
||||||
|
LanguageProviderProperties(listOf(PAPAGO), listOf(PAPAGO)),
|
||||||
|
memoryRepository
|
||||||
|
),
|
||||||
|
materializer = materializer,
|
||||||
|
transactionManager = TestTransactionManager()
|
||||||
|
)
|
||||||
|
val job = translationJob().also {
|
||||||
|
it.id = 200L
|
||||||
|
it.retryCount = 5
|
||||||
|
it.lastErrorMessage = "previous failure"
|
||||||
|
}
|
||||||
|
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
|
||||||
|
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
|
||||||
|
|
||||||
|
assertTrue(worker.processNextGroup())
|
||||||
|
|
||||||
|
assertEquals(TranslationJobStatus.COMPLETED, job.status)
|
||||||
|
assertEquals(5, job.retryCount)
|
||||||
|
assertNull(job.lastErrorMessage)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun translationJob(
|
private fun translationJob(
|
||||||
|
|||||||
Reference in New Issue
Block a user