Test #471

Merged
klaus merged 6 commits from test into main 2026-11-16 13:21:57 +00:00
5 changed files with 382 additions and 13 deletions
Showing only changes of commit 3ed79f2acb - Show all commits
@@ -1,6 +1,7 @@
package kr.co.vividnext.sodalive.i18n.translation
import org.springframework.data.jpa.repository.JpaRepository
import org.springframework.data.jpa.repository.Modifying
import org.springframework.data.jpa.repository.Query
import org.springframework.data.repository.query.Param
import java.time.LocalDateTime
@@ -14,6 +15,27 @@ interface TranslationJobRepository : JpaRepository<TranslationJob, Long> {
sourceHash: String
): TranslationJob?
fun findByResourceTypeAndResourceIdInAndTargetLanguageAndStatus(
resourceType: LanguageTranslationTargetType,
resourceIds: Collection<Long>,
targetLanguage: String,
status: TranslationJobStatus
): List<TranslationJob>
@Modifying
@Query(
"""
update TranslationJob j
set j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.PENDING,
j.retryCount = 0,
j.lastErrorMessage = null,
j.nextRetryAt = :now
where j.id = :jobId
and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.FAILED
"""
)
fun restartFailedJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int
@Query(
"""
select j from TranslationJob j
@@ -34,7 +34,12 @@ class TranslationJobScheduler(
targetLanguage = normalizedTargetLanguage,
sourceHash = sourceHash
)
if (existingJob != null) return
if (existingJob != null) {
if (existingJob.status == TranslationJobStatus.FAILED && existingJob.sourceLanguage == normalizedSourceLanguage) {
translationJobRepository.restartFailedJob(existingJob.id!!, LocalDateTime.now())
}
return
}
translationJobRepository.save(
TranslationJob(
@@ -18,7 +18,7 @@ class TranslationJobWorker(
private val log = LoggerFactory.getLogger(javaClass)
private val transactionTemplate = TransactionTemplate(transactionManager)
@Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:600000}")
@Scheduled(fixedDelayString = "\${sodalive.translation-job.fixed-delay-ms:1000}")
fun runPendingJobs() {
repeat(MAX_GROUPS_PER_TICK) {
if (!processNextGroup()) return
@@ -121,7 +121,7 @@ class TranslationJobWorker(
val job = translationJobRepository.findById(jobId).orElse(null) ?: return@executeWithoutResult
job.retryCount += 1
job.lastErrorMessage = ex.message?.take(MAX_ERROR_LENGTH)
if (job.retryCount >= MAX_RETRY_COUNT) {
if (job.retryCount >= MAX_ATTEMPTS) {
job.status = TranslationJobStatus.FAILED
} else {
job.status = TranslationJobStatus.PENDING
@@ -142,6 +142,6 @@ class TranslationJobWorker(
companion object {
private const val MAX_GROUPS_PER_TICK = 5
private const val MAX_ERROR_LENGTH = 1000
private const val MAX_RETRY_COUNT = 3
private const val MAX_ATTEMPTS = 6
}
}
@@ -0,0 +1,294 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.configs.QueryDslConfig
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertFalse
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.EnumSource
import org.junit.jupiter.params.provider.ValueSource
import org.mockito.AdditionalAnswers
import org.mockito.Mockito
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
import org.springframework.context.annotation.Import
import org.springframework.data.jpa.repository.Query
import org.springframework.transaction.PlatformTransactionManager
import org.springframework.transaction.annotation.Propagation
import org.springframework.transaction.annotation.Transactional
import org.springframework.transaction.support.TransactionTemplate
import java.time.LocalDateTime
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import javax.persistence.EntityManager
@DataJpaTest(
properties = [
"spring.cache.type=none",
"spring.datasource.url=jdbc:h2:mem:translation-job-retry;MODE=MySQL;NON_KEYWORDS=VALUE;DB_CLOSE_ON_EXIT=FALSE"
]
)
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
@Import(QueryDslConfig::class, TranslationJobScheduler::class)
class TranslationJobRetryIntegrationTest @Autowired constructor(
private val repository: TranslationJobRepository,
private val scheduler: TranslationJobScheduler,
private val entityManager: EntityManager,
private val transactionManager: PlatformTransactionManager
) {
private val transaction = TransactionTemplate(transactionManager)
private val requiresNewTransaction = TransactionTemplate(transactionManager).apply {
propagationBehavior = Propagation.REQUIRES_NEW.value()
}
private val previousRetryAt = LocalDateTime.of(2040, 1, 1, 0, 0)
@ParameterizedTest
@ValueSource(ints = [3, 6])
@DisplayName("기존 실패 횟수가 3회 또는 6회인 FAILED 행을 같은 원문 요청으로 즉시 재개한다")
fun shouldResetSameFailedRowForCurrentNormalizedSource(previousFailures: Int) {
val original = persistJob(retryCount = previousFailures)
val requestStartedAt = LocalDateTime.now().withNano(0)
schedule()
val requestFinishedAt = LocalDateTime.now().plusSeconds(1).withNano(0)
entityManager.flush()
entityManager.clear()
val resumed = repository.findAll().single()
assertEquals(original.id, resumed.id)
assertEquals("제목", resumed.sourceText)
assertEquals(SourceTextNormalizer.hash("제목"), resumed.sourceHash)
assertEquals(TranslationJobStatus.PENDING, resumed.status)
assertEquals(0, resumed.retryCount)
assertNull(resumed.lastErrorMessage)
assertTrue(!resumed.nextRetryAt.isBefore(requestStartedAt))
assertTrue(resumed.nextRetryAt.isBefore(requestFinishedAt))
}
@ParameterizedTest
@EnumSource(TranslationJobStatus::class, names = ["PENDING", "RUNNING", "COMPLETED"])
@DisplayName("반복 요청은 진행 중이거나 완료된 행의 상태와 재시도 예산을 바꾸지 않는다")
fun shouldPreserveExistingRowOutsideFailedStatus(status: TranslationJobStatus) {
val original = persistJob(status = status, retryCount = 4)
repeat(2) { schedule() }
entityManager.flush()
entityManager.clear()
val preserved = repository.findAll().single()
assertEquals(original.id, preserved.id)
assertEquals(status, preserved.status)
assertEquals(4, preserved.retryCount)
assertEquals("previous failure", preserved.lastErrorMessage)
assertEquals(previousRetryAt, preserved.nextRetryAt)
}
@Test
@DisplayName("같은 원문 해시라도 원문 언어가 다른 FAILED 행은 재개하지 않는다")
fun shouldNotRestartFailedJobForDifferentSourceLanguage() {
val original = persistJob(sourceLanguage = "ja")
schedule()
entityManager.flush()
entityManager.clear()
val preserved = repository.findAll().single()
assertEquals(original.id, preserved.id)
assertEquals("ja", preserved.sourceLanguage)
assertEquals(TranslationJobStatus.FAILED, preserved.status)
assertEquals(6, preserved.retryCount)
assertEquals("previous failure", preserved.lastErrorMessage)
assertEquals(previousRetryAt, preserved.nextRetryAt)
}
@Test
@DisplayName("현재 원문 요청은 이전 원문의 FAILED 행을 재개하지 않고 현재 해시의 작업을 생성한다")
fun shouldLeaveFailedRowForPreviousSourceUntouched() {
val previous = persistJob(sourceText = "이전 제목")
schedule()
entityManager.flush()
entityManager.clear()
val jobs = repository.findAll()
assertEquals(2, jobs.size)
val unchanged = jobs.single { it.id == previous.id }
assertEquals(TranslationJobStatus.FAILED, unchanged.status)
assertEquals(6, unchanged.retryCount)
assertEquals("previous failure", unchanged.lastErrorMessage)
assertEquals(previousRetryAt, unchanged.nextRetryAt)
val current = jobs.single { it.sourceHash == SourceTextNormalizer.hash("제목") }
assertEquals(TranslationJobStatus.PENDING, current.status)
assertEquals(0, current.retryCount)
assertNull(current.lastErrorMessage)
}
@Test
@DisplayName("재시도 시각이 지난 FAILED 행도 요청 없이 워커가 자동 선점하지 않는다")
fun shouldNotPollFailedRowWithoutNewRequest() {
val original = persistJob().apply { nextRetryAt = LocalDateTime.of(2000, 1, 1, 0, 0) }
entityManager.flush()
val claimSql = TranslationJobRepository::class.java
.getMethod("findNextPendingGroupJobIdsForUpdate", LocalDateTime::class.java)
.getAnnotation(Query::class.java).value
.replace("skip locked", "") // H2에서는 잠금 옵션만 제외하고 실제 운영 쿼리의 선점 조건을 실행한다.
val claimRepository = Mockito.mock(
TranslationJobRepository::class.java,
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
)
Mockito.doAnswer { call ->
entityManager.createNativeQuery(claimSql)
.setParameter("now", call.getArgument<LocalDateTime>(0))
.resultList.map { (it as Number).toLong() }
}.`when`(claimRepository).findNextPendingGroupJobIdsForUpdate(
Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now()
)
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
val provider = object : TranslationProvider {
override val providerId = LanguageProviderId.PAPAGO
override val providerVersion = "test"
override fun translate(request: TranslateRequest): TranslateResult =
throw IllegalStateException("unexpected translation")
}
val worker = TranslationJobWorker(
claimRepository,
memoryRepository,
TranslationCoordinator(
listOf(provider),
LanguageProviderProperties(listOf(LanguageProviderId.PAPAGO), listOf(LanguageProviderId.PAPAGO)),
memoryRepository
),
Mockito.mock(TranslationReadModelMaterializer::class.java),
transactionManager
)
assertFalse(worker.processNextGroup())
entityManager.flush()
entityManager.clear()
val untouched = repository.findAll().single()
assertEquals(original.id, untouched.id)
assertEquals(TranslationJobStatus.FAILED, untouched.status)
assertEquals(6, untouched.retryCount)
assertEquals("previous failure", untouched.lastErrorMessage)
assertEquals(LocalDateTime.of(2000, 1, 1, 0, 0), untouched.nextRetryAt)
}
@Test
@Transactional(propagation = Propagation.NOT_SUPPORTED)
@DisplayName("같은 FAILED 행의 동시 재개 요청은 중복 행 없이 하나의 새 주기만 연다")
fun shouldResumeSameFailedRowFromConcurrentRequests() {
val original = persistJob()
val originalId = requireNotNull(original.id)
val executor = Executors.newFixedThreadPool(2)
val ready = CountDownLatch(2)
val start = CountDownLatch(1)
try {
val requests = (1..2).map {
executor.submit {
ready.countDown()
check(start.await(5, TimeUnit.SECONDS)) { "concurrent request start timed out" }
schedule()
}
}
assertTrue(ready.await(5, TimeUnit.SECONDS))
start.countDown()
requests.forEach { it.get(5, TimeUnit.SECONDS) }
val resumed = repository.findAll().single()
assertEquals(originalId, resumed.id)
assertEquals(TranslationJobStatus.PENDING, resumed.status)
assertEquals(0, resumed.retryCount)
assertNull(resumed.lastErrorMessage)
} finally {
start.countDown()
executor.shutdownNow()
check(executor.awaitTermination(5, TimeUnit.SECONDS)) { "concurrent requests did not stop" }
repository.deleteById(originalId)
}
}
@Test
@Transactional(propagation = Propagation.NOT_SUPPORTED)
@DisplayName("FAILED 조회 뒤 다른 트랜잭션이 RUNNING으로 선점하면 오래된 재개 요청은 예산을 초기화하지 않는다")
fun shouldPreserveClaimCommittedAfterFailedSnapshotWasRead() {
val originalId = requireNotNull(persistJob().id)
val interleavingRepository = Mockito.mock(
TranslationJobRepository::class.java,
AdditionalAnswers.delegatesTo<TranslationJobRepository>(repository)
)
Mockito.doAnswer {
val snapshot = repository.findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
LanguageTranslationTargetType.CONTENT,
10L,
"title",
"en",
SourceTextNormalizer.hash("제목")
)
requiresNewTransaction.executeWithoutResult {
val claimed = repository.findById(originalId).orElseThrow()
claimed.status = TranslationJobStatus.RUNNING
claimed.retryCount = 4
repository.saveAndFlush(claimed)
}
snapshot
}.`when`(interleavingRepository).findByResourceTypeAndResourceIdAndFieldKeyAndTargetLanguageAndSourceHash(
LanguageTranslationTargetType.CONTENT,
10L,
"title",
"en",
SourceTextNormalizer.hash("제목")
)
try {
transaction.executeWithoutResult {
schedule(TranslationJobScheduler(interleavingRepository))
}
val claimed = repository.findAll().single()
assertEquals(originalId, claimed.id)
assertEquals(TranslationJobStatus.RUNNING, claimed.status)
assertEquals(4, claimed.retryCount)
assertEquals("previous failure", claimed.lastErrorMessage)
assertEquals(previousRetryAt, claimed.nextRetryAt)
} finally {
repository.deleteById(originalId)
}
}
private fun schedule(scheduler: TranslationJobScheduler = this.scheduler) {
scheduler.scheduleMissingTranslation(
resourceType = LanguageTranslationTargetType.CONTENT,
resourceId = 10L,
fieldKey = "title",
sourceText = " 제목\n",
sourceLanguage = "ko",
targetLanguage = "EN"
)
}
private fun persistJob(
status: TranslationJobStatus = TranslationJobStatus.FAILED,
sourceText: String = "제목",
retryCount: Int = 6,
sourceLanguage: String = "ko"
): TranslationJob = repository.saveAndFlush(
TranslationJob(
resourceType = LanguageTranslationTargetType.CONTENT,
resourceId = 10L,
fieldKey = "title",
sourceHash = SourceTextNormalizer.hash(sourceText),
sourceText = sourceText,
sourceLanguage = sourceLanguage,
targetLanguage = "en",
status = status,
retryCount = retryCount,
lastErrorMessage = "previous failure",
nextRetryAt = previousRetryAt
)
)
}
@@ -2,8 +2,12 @@ package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderId.PAPAGO
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.CsvSource
import org.mockito.ArgumentCaptor
import org.mockito.Mockito
import org.springframework.scheduling.annotation.Scheduled
@@ -14,12 +18,13 @@ import java.util.Optional
class TranslationJobWorkerTest {
@Test
fun shouldRunEveryTenMinutesByDefault() {
@DisplayName("번역 워커의 기본 실행 간격은 1초다")
fun shouldRunEverySecondByDefault() {
val scheduled = TranslationJobWorker::class.java
.getDeclaredMethod("runPendingJobs")
.getAnnotation(Scheduled::class.java)
assertEquals("\${sodalive.translation-job.fixed-delay-ms:600000}", scheduled.fixedDelayString)
assertEquals("\${sodalive.translation-job.fixed-delay-ms:1000}", scheduled.fixedDelayString)
}
@Test
@@ -210,8 +215,14 @@ class TranslationJobWorkerTest {
assertEquals(setOf(1L, 2L, 3L, 4L, 5L), completedResourceIds)
}
@Test
fun shouldRetryFailedJobWithBackoff() {
@ParameterizedTest(name = "기존 실패 {0}회 뒤 실패하면 {1}, 대기 {2}분")
@CsvSource("0, PENDING, 1", "1, PENDING, 5", "2, PENDING, 15", "3, PENDING, 15", "4, PENDING, 15", "5, FAILED, 0")
@DisplayName("최초 시도와 재시도 5회의 실패 횟수를 누적하고 여섯 번째 실패에 종료한다")
fun shouldScheduleFiveRetriesBeforeFailingOnSixthAttempt(
failuresBeforeAttempt: Int,
expectedStatus: TranslationJobStatus,
delayMinutes: Long
) {
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
val provider = failingProvider()
@@ -229,17 +240,54 @@ class TranslationJobWorkerTest {
)
val job = translationJob()
job.id = 200L
val beforeRetryAt = job.nextRetryAt
job.retryCount = failuresBeforeAttempt
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
worker.processNextGroup()
val attemptStartedAt = LocalDateTime.now()
assertTrue(worker.processNextGroup())
val attemptFinishedAt = LocalDateTime.now()
assertEquals(TranslationJobStatus.PENDING, job.status)
assertEquals(1, job.retryCount)
assertEquals(expectedStatus, job.status, "failure #${failuresBeforeAttempt + 1}")
assertEquals(failuresBeforeAttempt + 1, job.retryCount)
assertEquals("all translation providers failed", job.lastErrorMessage)
assertTrue(job.nextRetryAt.isAfter(beforeRetryAt))
if (expectedStatus == TranslationJobStatus.PENDING) {
assertTrue(!job.nextRetryAt.isBefore(attemptStartedAt.plusMinutes(delayMinutes)))
assertTrue(!job.nextRetryAt.isAfter(attemptFinishedAt.plusMinutes(delayMinutes)))
}
}
@Test
@DisplayName("다섯 번 실패한 작업도 여섯 번째 시도가 성공하면 완료된다")
fun shouldCompleteSixthAttemptAfterFiveFailures() {
val jobRepository = Mockito.mock(TranslationJobRepository::class.java)
val memoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
val materializer = Mockito.mock(TranslationReadModelMaterializer::class.java)
val worker = TranslationJobWorker(
translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository,
translationCoordinator = TranslationCoordinator(
listOf(successfulProvider()),
LanguageProviderProperties(listOf(PAPAGO), listOf(PAPAGO)),
memoryRepository
),
materializer = materializer,
transactionManager = TestTransactionManager()
)
val job = translationJob().also {
it.id = 200L
it.retryCount = 5
it.lastErrorMessage = "previous failure"
}
Mockito.`when`(jobRepository.findNextPendingGroupJobIdsForUpdate(anyLocalDateTime())).thenReturn(listOf(200L))
Mockito.`when`(jobRepository.findById(200L)).thenReturn(Optional.of(job))
assertTrue(worker.processNextGroup())
assertEquals(TranslationJobStatus.COMPLETED, job.status)
assertEquals(5, job.retryCount)
assertNull(job.lastErrorMessage)
}
private fun translationJob(