fix(i18n): 콘텐츠 태그 미번역을 Papago로 전환하고 완료 작업을 복구한다
This commit is contained in:
+22
-6
@@ -53,18 +53,34 @@ class ResourceTranslationJobScheduler(
|
||||
resourceIds,
|
||||
target,
|
||||
TranslationJobStatus.FAILED
|
||||
)
|
||||
) + if (resourceType == LanguageTranslationTargetType.CONTENT) {
|
||||
translationJobRepository.findCompletedContentTagJobs(resourceIds, target)
|
||||
} else {
|
||||
emptyList()
|
||||
}
|
||||
val now = LocalDateTime.now()
|
||||
candidates.groupBy { it.resourceId }.forEach { (resourceId, failedJobs) ->
|
||||
candidates.groupBy { it.resourceId }.forEach { (resourceId, resourceJobs) ->
|
||||
val source = sourceExtractor.extract(resourceType, resourceId) ?: return@forEach
|
||||
val sourceLanguage = source.sourceLanguage.lowercase()
|
||||
if (!TranslationLanguages.isSupported(sourceLanguage) || sourceLanguage == target) return@forEach
|
||||
|
||||
val sourceHashes = source.fields.associate { it.fieldKey to SourceTextNormalizer.hash(it.sourceText) }
|
||||
failedJobs.filter { job ->
|
||||
job.sourceLanguage == sourceLanguage && job.sourceHash == sourceHashes[job.fieldKey]
|
||||
val sourceFields = source.fields.associateBy { it.fieldKey }
|
||||
resourceJobs.filter { job ->
|
||||
job.sourceLanguage == sourceLanguage &&
|
||||
job.sourceHash == sourceFields[job.fieldKey]?.let { SourceTextNormalizer.hash(it.sourceText) }
|
||||
}.forEach { job ->
|
||||
translationJobRepository.restartFailedJob(requireNotNull(job.id), now)
|
||||
when (job.status) {
|
||||
TranslationJobStatus.FAILED -> translationJobRepository.restartFailedJob(requireNotNull(job.id), now)
|
||||
TranslationJobStatus.COMPLETED -> translationJobScheduler.scheduleMissingTranslation(
|
||||
resourceType = resourceType,
|
||||
resourceId = resourceId,
|
||||
fieldKey = job.fieldKey,
|
||||
sourceText = sourceFields.getValue(job.fieldKey).sourceText,
|
||||
sourceLanguage = sourceLanguage,
|
||||
targetLanguage = target
|
||||
)
|
||||
TranslationJobStatus.PENDING, TranslationJobStatus.RUNNING -> Unit
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,4 +20,8 @@ object SourceTextNormalizer {
|
||||
.digest(normalized.toByteArray(Charsets.UTF_8))
|
||||
return digest.joinToString("") { "%02x".format(it) }
|
||||
}
|
||||
|
||||
fun hasSameCommaSeparatedText(first: String, second: String): Boolean {
|
||||
return first.split(',').map(::normalize) == second.split(',').map(::normalize)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@ package kr.co.vividnext.sodalive.i18n.translation
|
||||
data class TranslateRequest(
|
||||
val texts: List<String>,
|
||||
val sourceLanguage: String,
|
||||
val targetLanguage: String
|
||||
val targetLanguage: String,
|
||||
val rejectUnchangedAwsResult: Boolean = false
|
||||
)
|
||||
|
||||
data class TranslateResult(
|
||||
|
||||
+22
-2
@@ -10,7 +10,12 @@ class TranslationCoordinator(
|
||||
) {
|
||||
private val providers = properties.orderedProviders(properties.translationOrder, providers) { it.providerId }
|
||||
|
||||
fun findCached(sourceHash: String, sourceLanguage: String, targetLanguage: String): TranslationMemory? {
|
||||
fun findCached(
|
||||
sourceHash: String,
|
||||
sourceLanguage: String,
|
||||
targetLanguage: String,
|
||||
rejectUnchangedAwsResult: Boolean = false
|
||||
): TranslationMemory? {
|
||||
providers.forEach { provider ->
|
||||
val memory = translationMemoryRepository
|
||||
.findBySourceHashAndSourceLanguageAndTargetLanguageAndProviderAndNormalizationVersion(
|
||||
@@ -20,7 +25,14 @@ class TranslationCoordinator(
|
||||
provider.providerId.code,
|
||||
SourceTextNormalizer.NORMALIZATION_VERSION
|
||||
)
|
||||
if (memory != null && memory.translatedText.isNotBlank()) return memory
|
||||
if (memory == null || memory.translatedText.isBlank()) return@forEach
|
||||
if (
|
||||
rejectUnchangedAwsResult && provider.providerId == LanguageProviderId.AWS &&
|
||||
SourceTextNormalizer.hasSameCommaSeparatedText(memory.sourceText, memory.translatedText)
|
||||
) {
|
||||
return@forEach
|
||||
}
|
||||
return memory
|
||||
}
|
||||
return null
|
||||
}
|
||||
@@ -42,6 +54,14 @@ class TranslationCoordinator(
|
||||
result != null && result.translatedText.size == request.texts.size &&
|
||||
result.translatedText.none { it.isBlank() }
|
||||
) {
|
||||
if (
|
||||
request.rejectUnchangedAwsResult && provider.providerId == LanguageProviderId.AWS &&
|
||||
request.texts.zip(result.translatedText).any { (source, translated) ->
|
||||
SourceTextNormalizer.hasSameCommaSeparatedText(source, translated)
|
||||
}
|
||||
) {
|
||||
return@forEach
|
||||
}
|
||||
return ProviderTranslationResult(result, provider.providerId, provider.providerVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,21 @@ interface TranslationJobRepository : JpaRepository<TranslationJob, Long> {
|
||||
status: TranslationJobStatus
|
||||
): List<TranslationJob>
|
||||
|
||||
@Query(
|
||||
"""
|
||||
select j from TranslationJob j
|
||||
where j.resourceType = kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType.CONTENT
|
||||
and j.resourceId in :resourceIds
|
||||
and j.targetLanguage = :targetLanguage
|
||||
and j.fieldKey = 'tags'
|
||||
and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.COMPLETED
|
||||
"""
|
||||
)
|
||||
fun findCompletedContentTagJobs(
|
||||
@Param("resourceIds") resourceIds: Collection<Long>,
|
||||
@Param("targetLanguage") targetLanguage: String
|
||||
): List<TranslationJob>
|
||||
|
||||
@Modifying
|
||||
@Query(
|
||||
"""
|
||||
@@ -36,6 +51,22 @@ interface TranslationJobRepository : JpaRepository<TranslationJob, Long> {
|
||||
)
|
||||
fun restartFailedJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int
|
||||
|
||||
@Modifying
|
||||
@Query(
|
||||
"""
|
||||
update TranslationJob j
|
||||
set j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.PENDING,
|
||||
j.retryCount = 0,
|
||||
j.lastErrorMessage = null,
|
||||
j.nextRetryAt = :now
|
||||
where j.id = :jobId
|
||||
and j.resourceType = kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType.CONTENT
|
||||
and j.fieldKey = 'tags'
|
||||
and j.status = kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus.COMPLETED
|
||||
"""
|
||||
)
|
||||
fun restartCompletedContentTagsJob(@Param("jobId") jobId: Long, @Param("now") now: LocalDateTime): Int
|
||||
|
||||
@Query(
|
||||
"""
|
||||
select j from TranslationJob j
|
||||
|
||||
+32
-3
@@ -1,12 +1,15 @@
|
||||
package kr.co.vividnext.sodalive.i18n.translation
|
||||
|
||||
import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository
|
||||
import org.springframework.stereotype.Service
|
||||
import org.springframework.transaction.annotation.Transactional
|
||||
import java.time.LocalDateTime
|
||||
|
||||
@Service
|
||||
class TranslationJobScheduler(
|
||||
private val translationJobRepository: TranslationJobRepository
|
||||
private val translationJobRepository: TranslationJobRepository,
|
||||
private val translationCoordinator: TranslationCoordinator,
|
||||
private val contentTranslationRepository: ContentTranslationRepository
|
||||
) {
|
||||
@Transactional
|
||||
fun scheduleMissingTranslation(
|
||||
@@ -35,8 +38,34 @@ class TranslationJobScheduler(
|
||||
sourceHash = sourceHash
|
||||
)
|
||||
if (existingJob != null) {
|
||||
if (existingJob.status == TranslationJobStatus.FAILED && existingJob.sourceLanguage == normalizedSourceLanguage) {
|
||||
translationJobRepository.restartFailedJob(existingJob.id!!, LocalDateTime.now())
|
||||
if (existingJob.sourceLanguage != normalizedSourceLanguage) return
|
||||
|
||||
when (existingJob.status) {
|
||||
TranslationJobStatus.FAILED ->
|
||||
translationJobRepository.restartFailedJob(requireNotNull(existingJob.id), LocalDateTime.now())
|
||||
TranslationJobStatus.COMPLETED -> {
|
||||
if (resourceType != LanguageTranslationTargetType.CONTENT || fieldKey != "tags") return
|
||||
|
||||
val renderedTags = contentTranslationRepository.findByContentIdAndLocale(resourceId, normalizedTargetLanguage)
|
||||
?.renderedPayload?.tags
|
||||
if (
|
||||
!renderedTags.isNullOrBlank() &&
|
||||
!SourceTextNormalizer.hasSameCommaSeparatedText(normalizedText, renderedTags)
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
val cached = translationCoordinator.findCached(
|
||||
sourceHash,
|
||||
normalizedSourceLanguage,
|
||||
normalizedTargetLanguage,
|
||||
rejectUnchangedAwsResult = true
|
||||
)
|
||||
if (cached != null && cached.translatedText == renderedTags) return
|
||||
|
||||
translationJobRepository.restartCompletedContentTagsJob(requireNotNull(existingJob.id), LocalDateTime.now())
|
||||
}
|
||||
TranslationJobStatus.PENDING, TranslationJobStatus.RUNNING -> Unit
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -69,14 +69,21 @@ class TranslationJobWorker(
|
||||
}
|
||||
|
||||
private fun ensureMemory(job: TranslationJob) {
|
||||
val existing = translationCoordinator.findCached(job.sourceHash, job.sourceLanguage, job.targetLanguage)
|
||||
val rejectUnchangedAwsResult = job.resourceType == LanguageTranslationTargetType.CONTENT && job.fieldKey == "tags"
|
||||
val existing = translationCoordinator.findCached(
|
||||
job.sourceHash,
|
||||
job.sourceLanguage,
|
||||
job.targetLanguage,
|
||||
rejectUnchangedAwsResult
|
||||
)
|
||||
if (existing != null) return
|
||||
|
||||
val response = translationCoordinator.translate(
|
||||
TranslateRequest(
|
||||
texts = listOf(job.sourceText),
|
||||
sourceLanguage = job.sourceLanguage,
|
||||
targetLanguage = job.targetLanguage
|
||||
targetLanguage = job.targetLanguage,
|
||||
rejectUnchangedAwsResult = rejectUnchangedAwsResult
|
||||
)
|
||||
)
|
||||
val translated = response.result.translatedText.single()
|
||||
@@ -102,6 +109,12 @@ class TranslationJobWorker(
|
||||
providerVersion = response.providerVersion
|
||||
)
|
||||
)
|
||||
} else if (
|
||||
rejectUnchangedAwsResult && response.providerId == LanguageProviderId.AWS &&
|
||||
SourceTextNormalizer.hasSameCommaSeparatedText(memory.sourceText, memory.translatedText)
|
||||
) {
|
||||
memory.translatedText = translated
|
||||
translationMemoryRepository.save(memory)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ class TranslationMemory(
|
||||
val targetLanguage: String,
|
||||
|
||||
@Column(name = "translated_text", nullable = false, columnDefinition = "text")
|
||||
val translatedText: String,
|
||||
var translatedText: String,
|
||||
|
||||
@Column(name = "provider", nullable = false, length = 50)
|
||||
val provider: String,
|
||||
|
||||
+3
-1
@@ -74,7 +74,9 @@ class TranslationReadModelMaterializer(
|
||||
val memory = translationCoordinator.findCached(
|
||||
sourceHash = SourceTextNormalizer.hash(normalizedText),
|
||||
sourceLanguage = source.sourceLanguage.lowercase(),
|
||||
targetLanguage = targetLanguage
|
||||
targetLanguage = targetLanguage,
|
||||
rejectUnchangedAwsResult = source.resourceType == LanguageTranslationTargetType.CONTENT &&
|
||||
field.fieldKey == "tags"
|
||||
) ?: return null
|
||||
|
||||
result[field.fieldKey] = memory.translatedText
|
||||
|
||||
Reference in New Issue
Block a user