AWS 번역 공급자 추가 #469

Merged
klaus merged 12 commits from test into main 2026-11-10 05:25:25 +00:00
16 changed files with 545 additions and 221 deletions
Showing only changes of commit f7fadf7ac7 - Show all commits
@@ -10,15 +10,11 @@ import kr.co.vividnext.sodalive.explorer.profile.CreatorCheersRepository
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository
import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationEvent import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationEvent
import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType
import kr.co.vividnext.sodalive.i18n.translation.PapagoTranslationService.Companion.getTranslatableLanguageCodes import kr.co.vividnext.sodalive.i18n.translation.TranslationLanguages.getTranslatableLanguageCodes
import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.application.CreatorCommunityTranslationService import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.application.CreatorCommunityTranslationService
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Value
import org.springframework.context.ApplicationEventPublisher import org.springframework.context.ApplicationEventPublisher
import org.springframework.data.repository.findByIdOrNull import org.springframework.data.repository.findByIdOrNull
import org.springframework.http.HttpEntity
import org.springframework.http.HttpHeaders
import org.springframework.http.MediaType
import org.springframework.scheduling.annotation.Async import org.springframework.scheduling.annotation.Async
import org.springframework.stereotype.Component import org.springframework.stereotype.Component
import org.springframework.transaction.annotation.Propagation import org.springframework.transaction.annotation.Propagation
@@ -27,13 +23,11 @@ import org.springframework.transaction.event.TransactionPhase
import org.springframework.transaction.event.TransactionalEventListener import org.springframework.transaction.event.TransactionalEventListener
import org.springframework.transaction.support.TransactionSynchronization import org.springframework.transaction.support.TransactionSynchronization
import org.springframework.transaction.support.TransactionSynchronizationManager import org.springframework.transaction.support.TransactionSynchronizationManager
import org.springframework.util.LinkedMultiValueMap
import org.springframework.web.client.RestTemplate
import javax.persistence.EntityManager import javax.persistence.EntityManager
import javax.persistence.LockModeType import javax.persistence.LockModeType
/** /**
* 텍스트 기반 데이터(콘텐츠, 댓글 등)에 대해 파파고 언어 감지를 요청하기 위한 이벤트. * 텍스트 기반 데이터(콘텐츠, 댓글 등)에 대해 언어 감지를 요청하기 위한 이벤트.
*/ */
enum class LanguageDetectTargetType { enum class LanguageDetectTargetType {
CONTENT, CONTENT,
@@ -56,10 +50,6 @@ class LanguageDetectEvent(
val targetLanguage: String? = null val targetLanguage: String? = null
) )
data class PapagoLanguageDetectResponse(
val langCode: String?
)
@Component @Component
class LanguageDetectListener( class LanguageDetectListener(
private val audioContentRepository: AudioContentRepository, private val audioContentRepository: AudioContentRepository,
@@ -75,27 +65,17 @@ class LanguageDetectListener(
private val creatorCommunityTranslationService: CreatorCommunityTranslationService, private val creatorCommunityTranslationService: CreatorCommunityTranslationService,
private val entityManager: EntityManager, private val entityManager: EntityManager,
private val applicationEventPublisher: ApplicationEventPublisher, private val applicationEventPublisher: ApplicationEventPublisher
@Value("\${cloud.naver.papago-client-id}")
private val papagoClientId: String,
@Value("\${cloud.naver.papago-client-secret}")
private val papagoClientSecret: String
) { ) {
private val log = LoggerFactory.getLogger(LanguageDetectListener::class.java) private val log = LoggerFactory.getLogger(LanguageDetectListener::class.java)
private val restTemplate: RestTemplate = RestTemplate()
private val papagoDetectUrl = "https://papago.apigw.ntruss.com/langs/v1/dect"
@Async @Async
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT)
@Transactional(propagation = Propagation.REQUIRES_NEW) @Transactional(propagation = Propagation.REQUIRES_NEW)
fun detectLanguage(event: LanguageDetectEvent) { fun detectLanguage(event: LanguageDetectEvent) {
if (event.query.isBlank()) { if (event.query.isBlank()) {
log.debug("[PapagoLanguageDetect] query is blank. Skip language detection. event={}", event) log.debug("[LanguageDetect] query is blank. Skip language detection. targetId={}", event.id)
return return
} }
@@ -117,21 +97,21 @@ class LanguageDetectListener(
val character = chatCharacterRepository.findById(characterId).orElse(null) val character = chatCharacterRepository.findById(characterId).orElse(null)
if (character == null) { if (character == null) {
log.warn("[PapagoLanguageDetect] ChatCharacter not found. characterId={}", characterId) log.warn("[LanguageDetect] ChatCharacter not found. characterId={}", characterId)
return return
} }
// 이미 언어 코드가 설정된 경우 호출하지 않음 // 이미 언어 코드가 설정된 경우 호출하지 않음
if (!character.languageCode.isNullOrBlank()) { if (!character.languageCode.isNullOrBlank()) {
log.debug( log.debug(
"[PapagoLanguageDetect] languageCode already set. Skip language detection. characterId={}, languageCode={}", "[LanguageDetect] languageCode already set. Skip language detection. characterId={}, languageCode={}",
characterId, characterId,
character.languageCode character.languageCode
) )
return return
} }
val langCode = detectLanguageCode(event, characterId) ?: return val langCode = detectLanguageCode(event) ?: return
character.languageCode = langCode character.languageCode = langCode
chatCharacterRepository.save(character) chatCharacterRepository.save(character)
@@ -144,7 +124,7 @@ class LanguageDetectListener(
) )
log.info( log.info(
"[PapagoLanguageDetect] languageCode updated from Papago. characterId={}, langCode={}", "[LanguageDetect] languageCode updated. characterId={}, langCode={}",
characterId, characterId,
langCode langCode
) )
@@ -155,21 +135,21 @@ class LanguageDetectListener(
val audioContent = audioContentRepository.findById(contentId).orElse(null) val audioContent = audioContentRepository.findById(contentId).orElse(null)
if (audioContent == null) { if (audioContent == null) {
log.warn("[PapagoLanguageDetect] AudioContent not found. contentId={}", contentId) log.warn("[LanguageDetect] AudioContent not found. contentId={}", contentId)
return return
} }
// 이미 언어 코드가 설정된 경우 호출하지 않음 // 이미 언어 코드가 설정된 경우 호출하지 않음
if (!audioContent.languageCode.isNullOrBlank()) { if (!audioContent.languageCode.isNullOrBlank()) {
log.debug( log.debug(
"[PapagoLanguageDetect] languageCode already set. Skip language detection. contentId={}, languageCode={}", "[LanguageDetect] languageCode already set. Skip language detection. contentId={}, languageCode={}",
contentId, contentId,
audioContent.languageCode audioContent.languageCode
) )
return return
} }
val langCode = detectLanguageCode(event, contentId) ?: return val langCode = detectLanguageCode(event) ?: return
audioContent.languageCode = langCode audioContent.languageCode = langCode
@@ -184,7 +164,7 @@ class LanguageDetectListener(
) )
log.info( log.info(
"[PapagoLanguageDetect] languageCode updated from Papago. contentId={}, langCode={}", "[LanguageDetect] languageCode updated. contentId={}, langCode={}",
contentId, contentId,
langCode langCode
) )
@@ -195,27 +175,27 @@ class LanguageDetectListener(
val comment = audioContentCommentRepository.findById(commentId).orElse(null) val comment = audioContentCommentRepository.findById(commentId).orElse(null)
if (comment == null) { if (comment == null) {
log.warn("[PapagoLanguageDetect] AudioContentComment not found. commentId={}", commentId) log.warn("[LanguageDetect] AudioContentComment not found. commentId={}", commentId)
return return
} }
// 이미 언어 코드가 설정된 경우 호출하지 않음 // 이미 언어 코드가 설정된 경우 호출하지 않음
if (!comment.languageCode.isNullOrBlank()) { if (!comment.languageCode.isNullOrBlank()) {
log.debug( log.debug(
"[PapagoLanguageDetect] languageCode already set. Skip language detection. commentId={}, languageCode={}", "[LanguageDetect] languageCode already set. Skip language detection. commentId={}, languageCode={}",
commentId, commentId,
comment.languageCode comment.languageCode
) )
return return
} }
val langCode = detectLanguageCode(event, commentId) ?: return val langCode = detectLanguageCode(event) ?: return
comment.languageCode = langCode comment.languageCode = langCode
audioContentCommentRepository.save(comment) audioContentCommentRepository.save(comment)
log.info( log.info(
"[PapagoLanguageDetect] languageCode updated from Papago. commentId={}, langCode={}", "[LanguageDetect] languageCode updated. commentId={}, langCode={}",
commentId, commentId,
langCode langCode
) )
@@ -226,14 +206,14 @@ class LanguageDetectListener(
val comment = characterCommentRepository.findById(commentId).orElse(null) val comment = characterCommentRepository.findById(commentId).orElse(null)
if (comment == null) { if (comment == null) {
log.warn("[PapagoLanguageDetect] CharacterComment not found. commentId={}", commentId) log.warn("[LanguageDetect] CharacterComment not found. commentId={}", commentId)
return return
} }
// 이미 언어 코드가 설정된 경우 호출하지 않음 // 이미 언어 코드가 설정된 경우 호출하지 않음
if (!comment.languageCode.isNullOrBlank()) { if (!comment.languageCode.isNullOrBlank()) {
log.debug( log.debug(
"[PapagoLanguageDetect] languageCode already set. Skip language detection. " + "[LanguageDetect] languageCode already set. Skip language detection. " +
"characterCommentId={}, languageCode={}", "characterCommentId={}, languageCode={}",
commentId, commentId,
comment.languageCode comment.languageCode
@@ -241,13 +221,13 @@ class LanguageDetectListener(
return return
} }
val langCode = detectLanguageCode(event, commentId) ?: return val langCode = detectLanguageCode(event) ?: return
comment.languageCode = langCode comment.languageCode = langCode
characterCommentRepository.save(comment) characterCommentRepository.save(comment)
log.info( log.info(
"[PapagoLanguageDetect] languageCode updated from Papago. characterCommentId={}, langCode={}", "[LanguageDetect] languageCode updated. characterCommentId={}, langCode={}",
commentId, commentId,
langCode langCode
) )
@@ -258,27 +238,27 @@ class LanguageDetectListener(
val cheers = creatorCheersRepository.findById(cheersId).orElse(null) val cheers = creatorCheersRepository.findById(cheersId).orElse(null)
if (cheers == null) { if (cheers == null) {
log.warn("[PapagoLanguageDetect] CreatorCheers not found. cheersId={}", cheersId) log.warn("[LanguageDetect] CreatorCheers not found. cheersId={}", cheersId)
return return
} }
// 이미 언어 코드가 설정된 경우 호출하지 않음 // 이미 언어 코드가 설정된 경우 호출하지 않음
if (!cheers.languageCode.isNullOrBlank()) { if (!cheers.languageCode.isNullOrBlank()) {
log.debug( log.debug(
"[PapagoLanguageDetect] languageCode already set. Skip language detection. cheersId={}, languageCode={}", "[LanguageDetect] languageCode already set. Skip language detection. cheersId={}, languageCode={}",
cheersId, cheersId,
cheers.languageCode cheers.languageCode
) )
return return
} }
val langCode = detectLanguageCode(event, cheersId) ?: return val langCode = detectLanguageCode(event) ?: return
cheers.languageCode = langCode cheers.languageCode = langCode
creatorCheersRepository.save(cheers) creatorCheersRepository.save(cheers)
log.info( log.info(
"[PapagoLanguageDetect] languageCode updated from Papago. cheersId={}, langCode={}", "[LanguageDetect] languageCode updated. cheersId={}, langCode={}",
cheersId, cheersId,
langCode langCode
) )
@@ -289,21 +269,21 @@ class LanguageDetectListener(
val series = seriesRepository.findByIdOrNull(seriesId) val series = seriesRepository.findByIdOrNull(seriesId)
if (series == null) { if (series == null) {
log.warn("[PapagoLanguageDetect] Series not found. seriesId={}", seriesId) log.warn("[LanguageDetect] Series not found. seriesId={}", seriesId)
return return
} }
// 이미 언어 코드가 설정된 경우 호출하지 않음 // 이미 언어 코드가 설정된 경우 호출하지 않음
if (!series.languageCode.isNullOrBlank()) { if (!series.languageCode.isNullOrBlank()) {
log.debug( log.debug(
"[PapagoLanguageDetect] languageCode already set. Skip language detection. seriesId={}, languageCode={}", "[LanguageDetect] languageCode already set. Skip language detection. seriesId={}, languageCode={}",
seriesId, seriesId,
series.languageCode series.languageCode
) )
return return
} }
val langCode = detectLanguageCode(event, seriesId) ?: return val langCode = detectLanguageCode(event) ?: return
series.languageCode = langCode series.languageCode = langCode
seriesRepository.save(series) seriesRepository.save(series)
@@ -316,7 +296,7 @@ class LanguageDetectListener(
) )
log.info( log.info(
"[PapagoLanguageDetect] languageCode updated from Papago. seriesId={}, langCode={}", "[LanguageDetect] languageCode updated. seriesId={}, langCode={}",
seriesId, seriesId,
langCode langCode
) )
@@ -327,21 +307,21 @@ class LanguageDetectListener(
val originalWork = originalWorkRepository.findByIdOrNull(originalWorkId) val originalWork = originalWorkRepository.findByIdOrNull(originalWorkId)
if (originalWork == null) { if (originalWork == null) {
log.warn("[PapagoLanguageDetect] OriginalWork not found. originalWorkId={}", originalWorkId) log.warn("[LanguageDetect] OriginalWork not found. originalWorkId={}", originalWorkId)
return return
} }
// 이미 언어 코드가 설정된 경우 호출하지 않음 // 이미 언어 코드가 설정된 경우 호출하지 않음
if (!originalWork.languageCode.isNullOrBlank()) { if (!originalWork.languageCode.isNullOrBlank()) {
log.debug( log.debug(
"[PapagoLanguageDetect] languageCode already set. Skip language detection. originalWorkId={}, languageCode={}", "[LanguageDetect] languageCode already set. Skip language detection. originalWorkId={}, languageCode={}",
originalWorkId, originalWorkId,
originalWork.languageCode originalWork.languageCode
) )
return return
} }
val langCode = detectLanguageCode(event, originalWorkId) ?: return val langCode = detectLanguageCode(event) ?: return
originalWork.languageCode = langCode originalWork.languageCode = langCode
originalWorkRepository.save(originalWork) originalWorkRepository.save(originalWork)
@@ -355,7 +335,7 @@ class LanguageDetectListener(
) )
log.info( log.info(
"[PapagoLanguageDetect] languageCode updated from Papago. originalWorkId={}, langCode={}", "[LanguageDetect] languageCode updated. originalWorkId={}, langCode={}",
originalWorkId, originalWorkId,
langCode langCode
) )
@@ -367,7 +347,7 @@ class LanguageDetectListener(
val category = categoryRepository.findByIdOrNull(categoryId) ?: return val category = categoryRepository.findByIdOrNull(categoryId) ?: return
if (!category.languageCode.isNullOrBlank()) return if (!category.languageCode.isNullOrBlank()) return
val langCode = detectLanguageCode(event, categoryId) ?: return val langCode = detectLanguageCode(event) ?: return
category.languageCode = langCode category.languageCode = langCode
categoryRepository.save(category) categoryRepository.save(category)
@@ -395,7 +375,7 @@ class LanguageDetectListener(
return return
} }
val languageCode = detectLanguageCode(event, event.id) ?: return val languageCode = detectLanguageCode(event) ?: return
if (languageCode !in getTranslatableLanguageCodes(null)) return if (languageCode !in getTranslatableLanguageCodes(null)) return
val currentCreatorCommunity = creatorCommunityRepository.findByIdAndIsActiveTrueForUpdate(event.id) ?: return val currentCreatorCommunity = creatorCommunityRepository.findByIdAndIsActiveTrueForUpdate(event.id) ?: return
@@ -427,60 +407,7 @@ class LanguageDetectListener(
) )
} }
private fun detectLanguageCode(event: LanguageDetectEvent, targetIdForLog: Long): String? { private fun detectLanguageCode(event: LanguageDetectEvent): String? {
return languageDetectionCacheService.detectWithCache(event.query) { return languageDetectionCacheService.detectWithCache(event.query)?.detectedLanguage
requestPapagoLanguageCode(event.query, targetIdForLog)
}
}
private fun requestPapagoLanguageCode(query: String, targetIdForLog: Long): String? {
return try {
val headers = HttpHeaders().apply {
contentType = MediaType.APPLICATION_FORM_URLENCODED
set("X-NCP-APIGW-API-KEY-ID", papagoClientId)
set("X-NCP-APIGW-API-KEY", papagoClientSecret)
}
val body = LinkedMultiValueMap<String, String>().apply {
// 파파고 스펙에 따라 query 파라미터에 텍스트를 공백으로 구분하여 전달
add("query", query)
}
val requestEntity = HttpEntity(body, headers)
val response = restTemplate.postForEntity(
papagoDetectUrl,
requestEntity,
PapagoLanguageDetectResponse::class.java
)
if (!response.statusCode.is2xxSuccessful) {
log.warn(
"[PapagoLanguageDetect] Non-success status from Papago. status={}, targetId={}",
response.statusCode,
targetIdForLog
)
return null
}
val langCode = response.body?.langCode?.takeIf { it.isNotBlank() }
if (langCode == null) {
log.warn(
"[PapagoLanguageDetect] langCode is null or blank in Papago response. targetId={}",
targetIdForLog
)
return null
}
langCode
} catch (ex: Exception) {
// 언어 감지는 부가 기능이므로, 실패 시 예외를 전파하지 않고 로그만 남긴다.
log.error(
"[PapagoLanguageDetect] Failed to detect language via Papago. targetId={}",
targetIdForLog,
ex
)
null
}
} }
} }
@@ -1,50 +1,64 @@
package kr.co.vividnext.sodalive.content package kr.co.vividnext.sodalive.content
import kr.co.vividnext.sodalive.i18n.translation.DetectedLanguage
import kr.co.vividnext.sodalive.i18n.translation.LanguageDetectionProvider
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderProperties
import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer
import org.springframework.stereotype.Service import org.springframework.stereotype.Service
import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.annotation.Transactional
@Service @Service
class LanguageDetectionCacheService( class LanguageDetectionCacheService(
private val languageDetectionResultRepository: LanguageDetectionResultRepository private val languageDetectionResultRepository: LanguageDetectionResultRepository,
providers: List<LanguageDetectionProvider>,
properties: LanguageProviderProperties
) { ) {
private val providers = properties.orderedProviders(properties.detectionOrder, providers) { it.providerName }
@Transactional @Transactional
fun detectWithCache( fun detectWithCache(query: String): DetectedLanguage? {
query: String,
provider: String = DEFAULT_PROVIDER,
detector: () -> String?
): String? {
val normalizedQuery = SourceTextNormalizer.normalize(query) val normalizedQuery = SourceTextNormalizer.normalize(query)
if (normalizedQuery.isBlank()) return null if (normalizedQuery.isBlank()) return null
val sourceHash = SourceTextNormalizer.hash(normalizedQuery) val sourceHash = SourceTextNormalizer.hash(normalizedQuery)
val cached = languageDetectionResultRepository.findBySourceHashAndProviderAndNormalizationVersion( providers.forEach { provider ->
sourceHash = sourceHash, val cached = languageDetectionResultRepository.findBySourceHashAndProviderAndNormalizationVersion(
provider = provider,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
)
if (cached != null) return cached.detectedLanguage
val detectedLanguage = detector()?.takeIf { it.isNotBlank() }?.lowercase() ?: return null
languageDetectionResultRepository.insertIfAbsent(
sourceHash = sourceHash,
sourceTextSample = normalizedQuery.take(MAX_SAMPLE_LENGTH),
detectedLanguage = detectedLanguage,
provider = provider,
confidence = null,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
)
return requireNotNull(
languageDetectionResultRepository.findDetectedLanguageForUpdate(
sourceHash = sourceHash, sourceHash = sourceHash,
provider = provider, provider = provider.providerName,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
) )
) if (cached != null && cached.detectedLanguage.isNotBlank()) {
return DetectedLanguage(cached.detectedLanguage, provider.providerName)
}
}
providers.forEach { provider ->
val detectedLanguage = try {
provider.detect(query)
} catch (_: Exception) {
null
}?.takeIf { it.isNotBlank() }?.lowercase() ?: return@forEach
languageDetectionResultRepository.insertIfAbsent(
sourceHash = sourceHash,
sourceTextSample = normalizedQuery.take(MAX_SAMPLE_LENGTH),
detectedLanguage = detectedLanguage,
provider = provider.providerName,
confidence = null,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
)
val winner = requireNotNull(
languageDetectionResultRepository.findDetectedLanguageForUpdate(
sourceHash = sourceHash,
provider = provider.providerName,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
)
)
return DetectedLanguage(winner, provider.providerName)
}
return null
} }
companion object { companion object {
const val DEFAULT_PROVIDER = "papago"
private const val MAX_SAMPLE_LENGTH = 500 private const val MAX_SAMPLE_LENGTH = 500
} }
} }
@@ -92,32 +92,10 @@ class PapagoTranslationService(
} }
private fun validateLanguages(sourceLanguage: String, targetLanguage: String): Boolean { private fun validateLanguages(sourceLanguage: String, targetLanguage: String): Boolean {
return requireSupportedLanguage(sourceLanguage) && requireSupportedLanguage(targetLanguage) return TranslationLanguages.isSupported(sourceLanguage) && TranslationLanguages.isSupported(targetLanguage)
}
private fun requireSupportedLanguage(language: String): Boolean {
val normalized = language.lowercase()
return SUPPORTED_LANGUAGE_CODES.contains(normalized)
} }
companion object { companion object {
private const val MAX_REQUEST_TEXT_LENGTH = 5000 private const val MAX_REQUEST_TEXT_LENGTH = 5000
private val SUPPORTED_LANGUAGE_CODES = setOf(
"ko",
"en",
"ja"
)
/**
* 번역 대상 언어 코드 집합을 반환한다.
*
* @param excludedLanguageCode 번역 대상에서 제외할 언어 코드(대소문자 무시)
* @return 지원되는 언어 코드 중 [excludedLanguageCode]를 제외한 집합
*/
fun getTranslatableLanguageCodes(excludedLanguageCode: String?): Set<String> {
val normalized = excludedLanguageCode?.lowercase()
return SUPPORTED_LANGUAGE_CODES.filterNot { it == normalized }.toSet()
}
} }
} }
@@ -1,6 +1,6 @@
package kr.co.vividnext.sodalive.i18n.translation package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.i18n.translation.PapagoTranslationService.Companion.getTranslatableLanguageCodes import kr.co.vividnext.sodalive.i18n.translation.TranslationLanguages.getTranslatableLanguageCodes
import org.springframework.stereotype.Service import org.springframework.stereotype.Service
import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.annotation.Transactional
@@ -21,7 +21,7 @@ class TranslationJobScheduler(
if (normalizedText.isBlank()) return if (normalizedText.isBlank()) return
val normalizedSourceLanguage = sourceLanguage.lowercase() val normalizedSourceLanguage = sourceLanguage.lowercase()
if (!SUPPORTED_SOURCE_LANGUAGE_CODES.contains(normalizedSourceLanguage)) return if (!TranslationLanguages.isSupported(normalizedSourceLanguage)) return
val normalizedTargetLanguage = targetLanguage.lowercase() val normalizedTargetLanguage = targetLanguage.lowercase()
if (normalizedSourceLanguage == normalizedTargetLanguage) return if (normalizedSourceLanguage == normalizedTargetLanguage) return
@@ -49,8 +49,4 @@ class TranslationJobScheduler(
) )
) )
} }
companion object {
private val SUPPORTED_SOURCE_LANGUAGE_CODES = setOf("ko", "en", "ja")
}
} }
@@ -11,7 +11,7 @@ import java.time.LocalDateTime
class TranslationJobWorker( class TranslationJobWorker(
private val translationJobRepository: TranslationJobRepository, private val translationJobRepository: TranslationJobRepository,
private val translationMemoryRepository: TranslationMemoryRepository, private val translationMemoryRepository: TranslationMemoryRepository,
private val translationProvider: TranslationProvider, private val translationCoordinator: TranslationCoordinator,
private val materializer: TranslationReadModelMaterializer, private val materializer: TranslationReadModelMaterializer,
transactionManager: PlatformTransactionManager transactionManager: PlatformTransactionManager
) { ) {
@@ -69,25 +69,17 @@ class TranslationJobWorker(
} }
private fun ensureMemory(job: TranslationJob) { private fun ensureMemory(job: TranslationJob) {
val existing = translationMemoryRepository val existing = translationCoordinator.findCached(job.sourceHash, job.sourceLanguage, job.targetLanguage)
.findBySourceHashAndSourceLanguageAndTargetLanguageAndProviderAndNormalizationVersion(
sourceHash = job.sourceHash,
sourceLanguage = job.sourceLanguage,
targetLanguage = job.targetLanguage,
provider = translationProvider.providerName,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
)
if (existing != null) return if (existing != null) return
val response = translationProvider.translate( val response = translationCoordinator.translate(
TranslateRequest( TranslateRequest(
texts = listOf(job.sourceText), texts = listOf(job.sourceText),
sourceLanguage = job.sourceLanguage, sourceLanguage = job.sourceLanguage,
targetLanguage = job.targetLanguage targetLanguage = job.targetLanguage
) )
) )
val translated = response.translatedText.firstOrNull()?.takeIf { it.isNotBlank() } val translated = response.result.translatedText.single()
?: throw IllegalStateException("empty translation result")
transactionTemplate.executeWithoutResult { transactionTemplate.executeWithoutResult {
val memory = translationMemoryRepository val memory = translationMemoryRepository
@@ -95,7 +87,7 @@ class TranslationJobWorker(
sourceHash = job.sourceHash, sourceHash = job.sourceHash,
sourceLanguage = job.sourceLanguage, sourceLanguage = job.sourceLanguage,
targetLanguage = job.targetLanguage, targetLanguage = job.targetLanguage,
provider = translationProvider.providerName, provider = response.providerName,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
) )
if (memory == null) { if (memory == null) {
@@ -106,8 +98,8 @@ class TranslationJobWorker(
sourceLanguage = job.sourceLanguage, sourceLanguage = job.sourceLanguage,
targetLanguage = job.targetLanguage, targetLanguage = job.targetLanguage,
translatedText = translated, translatedText = translated,
provider = translationProvider.providerName, provider = response.providerName,
providerVersion = translationProvider.providerVersion providerVersion = response.providerVersion
) )
) )
} }
@@ -29,7 +29,7 @@ import javax.persistence.LockModeType
@Service @Service
class TranslationReadModelMaterializer( class TranslationReadModelMaterializer(
private val sourceExtractor: TranslationSourceExtractor, private val sourceExtractor: TranslationSourceExtractor,
private val translationMemoryRepository: TranslationMemoryRepository, private val translationCoordinator: TranslationCoordinator,
private val contentTranslationRepository: ContentTranslationRepository, private val contentTranslationRepository: ContentTranslationRepository,
private val aiCharacterTranslationRepository: AiCharacterTranslationRepository, private val aiCharacterTranslationRepository: AiCharacterTranslationRepository,
private val contentThemeTranslationRepository: ContentThemeTranslationRepository, private val contentThemeTranslationRepository: ContentThemeTranslationRepository,
@@ -71,14 +71,11 @@ class TranslationReadModelMaterializer(
return@forEach return@forEach
} }
val memory = translationMemoryRepository val memory = translationCoordinator.findCached(
.findBySourceHashAndSourceLanguageAndTargetLanguageAndProviderAndNormalizationVersion( sourceHash = SourceTextNormalizer.hash(normalizedText),
sourceHash = SourceTextNormalizer.hash(normalizedText), sourceLanguage = source.sourceLanguage.lowercase(),
sourceLanguage = source.sourceLanguage.lowercase(), targetLanguage = targetLanguage
targetLanguage = targetLanguage, ) ?: return null
provider = DEFAULT_PROVIDER,
normalizationVersion = SourceTextNormalizer.NORMALIZATION_VERSION
) ?: return null
result[field.fieldKey] = memory.translatedText result[field.fieldKey] = memory.translatedText
} }
@@ -231,8 +228,4 @@ class TranslationReadModelMaterializer(
} }
return true return true
} }
companion object {
const val DEFAULT_PROVIDER = "papago"
}
} }
@@ -5,9 +5,9 @@ import kr.co.vividnext.sodalive.content.LanguageDetectTargetType
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunity import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunity
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository
import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType
import kr.co.vividnext.sodalive.i18n.translation.PapagoTranslationService.Companion.getTranslatableLanguageCodes
import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer
import kr.co.vividnext.sodalive.i18n.translation.TranslationLanguages.getTranslatableLanguageCodes
import kr.co.vividnext.sodalive.i18n.translation.TranslationReadModelMaterializer import kr.co.vividnext.sodalive.i18n.translation.TranslationReadModelMaterializer
import kr.co.vividnext.sodalive.i18n.translation.TranslationSourceExtractor import kr.co.vividnext.sodalive.i18n.translation.TranslationSourceExtractor
import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.adapter.out.persistence.CreatorCommunityTranslation import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.adapter.out.persistence.CreatorCommunityTranslation
@@ -15,9 +15,14 @@ import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.comment.Creato
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.like.CreatorCommunityLikeRepository import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.like.CreatorCommunityLikeRepository
import kr.co.vividnext.sodalive.i18n.LangContext import kr.co.vividnext.sodalive.i18n.LangContext
import kr.co.vividnext.sodalive.i18n.SodaMessageSource import kr.co.vividnext.sodalive.i18n.SodaMessageSource
import kr.co.vividnext.sodalive.i18n.translation.DetectedLanguage
import kr.co.vividnext.sodalive.i18n.translation.LanguageDetectionProvider
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderProperties
import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType
import kr.co.vividnext.sodalive.i18n.translation.PapagoTranslationService
import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer
import kr.co.vividnext.sodalive.i18n.translation.TranslationCoordinator
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.TranslationMemory import kr.co.vividnext.sodalive.i18n.translation.TranslationMemory
@@ -83,6 +88,9 @@ import javax.persistence.EntityManager
CreatorCommunityTranslationService::class, CreatorCommunityTranslationService::class,
TranslationSourceExtractor::class, TranslationSourceExtractor::class,
TranslationReadModelMaterializer::class, TranslationReadModelMaterializer::class,
TranslationCoordinator::class,
LanguageProviderProperties::class,
PapagoTranslationService::class,
ResourceTranslationJobScheduler::class, ResourceTranslationJobScheduler::class,
TranslationJobScheduler::class, TranslationJobScheduler::class,
CreatorCommunityLanguageDetectTest.AsyncConfiguration::class CreatorCommunityLanguageDetectTest.AsyncConfiguration::class
@@ -255,7 +263,7 @@ class CreatorCommunityLanguageDetectTest @Autowired constructor(
sourceLanguage = "en", sourceLanguage = "en",
targetLanguage = "ko", targetLanguage = "ko",
translatedText = "translated detail content", translatedText = "translated detail content",
provider = TranslationReadModelMaterializer.DEFAULT_PROVIDER, provider = "papago",
providerVersion = "nmt-v1" providerVersion = "nmt-v1"
) )
) )
@@ -719,7 +727,14 @@ class CreatorCommunityLanguageDetectTest @Autowired constructor(
class ControlledLanguageDetectionCacheService( class ControlledLanguageDetectionCacheService(
languageDetectionResultRepository: LanguageDetectionResultRepository languageDetectionResultRepository: LanguageDetectionResultRepository
) : LanguageDetectionCacheService(languageDetectionResultRepository) { ) : LanguageDetectionCacheService(
languageDetectionResultRepository,
listOf(object : LanguageDetectionProvider {
override val providerName = "papago"
override fun detect(query: String): String? = null
}),
LanguageProviderProperties()
) {
val detections = ConcurrentHashMap<String, String>() val detections = ConcurrentHashMap<String, String>()
val queries = ConcurrentLinkedQueue<String>() val queries = ConcurrentLinkedQueue<String>()
private val blockingDetections = ConcurrentHashMap<String, BlockingDetection>() private val blockingDetections = ConcurrentHashMap<String, BlockingDetection>()
@@ -740,19 +755,19 @@ class ControlledLanguageDetectionCacheService(
return ConcurrentDetection(detectedLanguage).also { concurrentDetections[query] = it } return ConcurrentDetection(detectedLanguage).also { concurrentDetections[query] = it }
} }
override fun detectWithCache(query: String, provider: String, detector: () -> String?): String? { override fun detectWithCache(query: String): DetectedLanguage? {
queries.add(query) queries.add(query)
val concurrentDetection = concurrentDetections[query] val concurrentDetection = concurrentDetections[query]
if (concurrentDetection != null) { if (concurrentDetection != null) {
return concurrentDetection.detect() return DetectedLanguage(concurrentDetection.detect(), "papago")
} }
val blockingDetection = blockingDetections[query] val blockingDetection = blockingDetections[query]
if (blockingDetection != null) { if (blockingDetection != null) {
blockingDetection.started.countDown() blockingDetection.started.countDown()
check(blockingDetection.release.await(5, TimeUnit.SECONDS)) check(blockingDetection.release.await(5, TimeUnit.SECONDS))
return blockingDetection.detectedLanguage return DetectedLanguage(blockingDetection.detectedLanguage, "papago")
} }
return detections[query] return detections[query]?.let { DetectedLanguage(it, "papago") }
} }
} }
@@ -4,9 +4,13 @@ import kr.co.vividnext.sodalive.configs.QueryDslConfig
import kr.co.vividnext.sodalive.content.theme.AudioContentThemeQueryRepository import kr.co.vividnext.sodalive.content.theme.AudioContentThemeQueryRepository
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunity import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunity
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderProperties
import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType
import kr.co.vividnext.sodalive.i18n.translation.PapagoLanguageDetectionProvider
import kr.co.vividnext.sodalive.i18n.translation.PapagoTranslationService
import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer
import kr.co.vividnext.sodalive.i18n.translation.TranslationCoordinator
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus import kr.co.vividnext.sodalive.i18n.translation.TranslationJobStatus
@@ -67,6 +71,10 @@ import java.util.concurrent.atomic.AtomicInteger
QueryDslConfig::class, QueryDslConfig::class,
AudioContentThemeQueryRepository::class, AudioContentThemeQueryRepository::class,
LanguageDetectionCacheService::class, LanguageDetectionCacheService::class,
LanguageProviderProperties::class,
PapagoLanguageDetectionProvider::class,
PapagoTranslationService::class,
TranslationCoordinator::class,
LanguageDetectListener::class, LanguageDetectListener::class,
CreatorCommunityTranslationService::class, CreatorCommunityTranslationService::class,
TranslationSourceExtractor::class, TranslationSourceExtractor::class,
@@ -82,6 +90,7 @@ class CreatorCommunityLanguageDetectionCacheConcurrencyTest @Autowired construct
private val translationJobRepository: TranslationJobRepository, private val translationJobRepository: TranslationJobRepository,
private val languageDetectionCacheService: LanguageDetectionCacheService, private val languageDetectionCacheService: LanguageDetectionCacheService,
private val languageDetectListener: LanguageDetectListener, private val languageDetectListener: LanguageDetectListener,
private val papagoLanguageDetectionProvider: PapagoLanguageDetectionProvider,
transactionManager: PlatformTransactionManager transactionManager: PlatformTransactionManager
) { ) {
private val requiresNewTransactionTemplate = TransactionTemplate(transactionManager).apply { private val requiresNewTransactionTemplate = TransactionTemplate(transactionManager).apply {
@@ -92,8 +101,7 @@ class CreatorCommunityLanguageDetectionCacheConcurrencyTest @Autowired construct
@BeforeEach @BeforeEach
fun setUp() { fun setUp() {
papagoRequests = ControlledPapagoRequestFactory() papagoRequests = ControlledPapagoRequestFactory()
val listener: LanguageDetectListener = AopTestUtils.getTargetObject(languageDetectListener) ReflectionTestUtils.setField(papagoLanguageDetectionProvider, "restTemplate", RestTemplate(papagoRequests))
ReflectionTestUtils.setField(listener, "restTemplate", RestTemplate(papagoRequests))
} }
@Test @Test
@@ -232,7 +240,7 @@ class CreatorCommunityLanguageDetectionCacheConcurrencyTest @Autowired construct
val rows = requiresNewTransactionTemplate.execute { val rows = requiresNewTransactionTemplate.execute {
languageDetectionResultRepository.findAll().filter { languageDetectionResultRepository.findAll().filter {
it.sourceHash == SourceTextNormalizer.hash(content) && it.sourceHash == SourceTextNormalizer.hash(content) &&
it.provider == LanguageDetectionCacheService.DEFAULT_PROVIDER && it.provider == "papago" &&
it.normalizationVersion == SourceTextNormalizer.NORMALIZATION_VERSION it.normalizationVersion == SourceTextNormalizer.NORMALIZATION_VERSION
} }
}.orEmpty() }.orEmpty()
@@ -1,5 +1,8 @@
package kr.co.vividnext.sodalive.content package kr.co.vividnext.sodalive.content
import kr.co.vividnext.sodalive.i18n.translation.DetectedLanguage
import kr.co.vividnext.sodalive.i18n.translation.LanguageDetectionProvider
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderProperties
import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer
import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.DisplayName
@@ -10,7 +13,8 @@ class LanguageDetectionCacheServiceTest {
@Test @Test
fun shouldReuseCachedLanguageDetectionForSameNormalizedText() { fun shouldReuseCachedLanguageDetectionForSameNormalizedText() {
val repository = Mockito.mock(LanguageDetectionResultRepository::class.java) val repository = Mockito.mock(LanguageDetectionResultRepository::class.java)
val service = LanguageDetectionCacheService(repository) var providerCalls = 0
val service = service(repository) { providerCalls++; "ko" }
val sourceHash = SourceTextNormalizer.hash("Hello world") val sourceHash = SourceTextNormalizer.hash("Hello world")
Mockito.`when`( Mockito.`when`(
@@ -30,13 +34,9 @@ class LanguageDetectionCacheServiceTest {
) )
) )
var providerCalls = 0 val detected = service.detectWithCache("Hello world")
val detected = service.detectWithCache("Hello world") {
providerCalls++
"ko"
}
assertEquals("en", detected) assertEquals(DetectedLanguage("en", "papago"), detected)
assertEquals(0, providerCalls) assertEquals(0, providerCalls)
Mockito.verify(repository).findBySourceHashAndProviderAndNormalizationVersion( Mockito.verify(repository).findBySourceHashAndProviderAndNormalizationVersion(
sourceHash = sourceHash, sourceHash = sourceHash,
@@ -50,7 +50,7 @@ class LanguageDetectionCacheServiceTest {
@DisplayName("최초 캐시 저장 경합에서는 저장된 승자의 감지 언어를 반환한다") @DisplayName("최초 캐시 저장 경합에서는 저장된 승자의 감지 언어를 반환한다")
fun shouldReturnStoredWinnerAfterInitialCacheInsert() { fun shouldReturnStoredWinnerAfterInitialCacheInsert() {
val repository = Mockito.mock(LanguageDetectionResultRepository::class.java) val repository = Mockito.mock(LanguageDetectionResultRepository::class.java)
val service = LanguageDetectionCacheService(repository) val service = service(repository) { "en" }
val sourceHash = SourceTextNormalizer.hash("Hello world") val sourceHash = SourceTextNormalizer.hash("Hello world")
Mockito.`when`( Mockito.`when`(
@@ -78,9 +78,9 @@ class LanguageDetectionCacheServiceTest {
) )
).thenReturn("ko") ).thenReturn("ko")
val detected = service.detectWithCache("Hello world") { "en" } val detected = service.detectWithCache("Hello world")
assertEquals("ko", detected) assertEquals(DetectedLanguage("ko", "papago"), detected)
Mockito.verify(repository).findBySourceHashAndProviderAndNormalizationVersion( Mockito.verify(repository).findBySourceHashAndProviderAndNormalizationVersion(
sourceHash = sourceHash, sourceHash = sourceHash,
provider = "papago", provider = "papago",
@@ -101,4 +101,12 @@ class LanguageDetectionCacheServiceTest {
) )
Mockito.verifyNoMoreInteractions(repository) Mockito.verifyNoMoreInteractions(repository)
} }
private fun service(repository: LanguageDetectionResultRepository, detect: () -> String?): LanguageDetectionCacheService {
val provider = object : LanguageDetectionProvider {
override val providerName = "papago"
override fun detect(query: String) = detect()
}
return LanguageDetectionCacheService(repository, listOf(provider), LanguageProviderProperties())
}
} }
@@ -0,0 +1,108 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.content.LanguageDetectionCacheService
import kr.co.vividnext.sodalive.content.LanguageDetectionResult
import kr.co.vividnext.sodalive.content.LanguageDetectionResultRepository
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertSame
import org.junit.jupiter.api.Assertions.assertThrows
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.mockito.Mockito
class LanguageDetectionProviderTest {
private val repository = Mockito.mock(LanguageDetectionResultRepository::class.java)
private val query = "original source"
private val hash = SourceTextNormalizer.hash(query)
@Test
@DisplayName("감지 예외와 빈 결과는 다음 공급자로 넘기고 실제 공급자의 저장 승자를 반환한다")
fun shouldFallbackAndReturnCanonicalWinnerWithActualProvider() {
val failures: List<() -> String?> = listOf({ throw IllegalStateException("sensitive body") }, { null }, { " " })
failures.forEach { failure ->
val first = RecordingDetectionProvider("first", failure)
val second = RecordingDetectionProvider("second") { "EN" }
Mockito.`when`(repository.findDetectedLanguageForUpdate(hash, "second", "v1")).thenReturn("ko")
assertEquals(DetectedLanguage("ko", "second"), service(listOf(first, second)).detectWithCache(query))
assertEquals(listOf(query), first.queries)
assertEquals(listOf(query), second.queries)
}
Mockito.verify(repository, Mockito.times(3)).insertIfAbsent(hash, "original source", "en", "second", null, "v1")
}
@Test
@DisplayName("보조 공급자의 유효 캐시가 있으면 첫 공급자도 호출하지 않는다")
fun shouldCheckAllConfiguredCachesBeforeCallingAnyProvider() {
val first = RecordingDetectionProvider("first") { "ko" }
val second = RecordingDetectionProvider("second") { "en" }
Mockito.`when`(repository.findBySourceHashAndProviderAndNormalizationVersion(hash, "second", "v1"))
.thenReturn(LanguageDetectionResult(hash, "sample", "ja", "second", null, "v1"))
assertEquals(DetectedLanguage("ja", "second"), service(listOf(first, second)).detectWithCache(query))
assertEquals(emptyList<String>(), first.queries + second.queries)
}
@Test
@DisplayName("정상 미지원 언어 감지는 성공이며 후속 공급자를 호출하지 않는다")
fun shouldKeepUnsupportedDetectionAsSuccessAndHonorOrder() {
val first = RecordingDetectionProvider("first") { "ko" }
val second = RecordingDetectionProvider("second") { "FR" }
Mockito.`when`(repository.findDetectedLanguageForUpdate(hash, "second", "v1")).thenReturn("fr")
assertEquals(
DetectedLanguage("fr", "second"),
service(listOf(first, second), listOf("second", "first")).detectWithCache(query)
)
assertEquals(emptyList<String>(), first.queries)
assertEquals(listOf(query), second.queries)
}
@Test
@DisplayName("감지 전체 실패와 공백 입력은 null이고 공백은 저장소와 공급자를 호출하지 않는다")
fun shouldReturnNullAfterAllFailuresAndSkipBlankInput() {
val first = RecordingDetectionProvider("first") { null }
val second = RecordingDetectionProvider("second") { null }
val service = service(listOf(first, second))
assertNull(service.detectWithCache(" \n"))
Mockito.verifyNoInteractions(repository)
assertNull(service.detectWithCache(query))
assertEquals(listOf(query), first.queries)
assertEquals(listOf(query), second.queries)
}
@Test
@DisplayName("감지 저장소 조회와 저장 오류는 다음 공급자로 전환하지 않고 전파한다")
fun shouldPropagateRepositoryFailuresWithoutFallback() {
val first = RecordingDetectionProvider("first") { "ko" }
val second = RecordingDetectionProvider("second") { "en" }
val service = service(listOf(first, second))
val failure = IllegalStateException("database unavailable")
Mockito.`when`(repository.findBySourceHashAndProviderAndNormalizationVersion(hash, "first", "v1"))
.thenThrow(failure)
assertSame(failure, assertThrows(IllegalStateException::class.java) { service.detectWithCache(query) })
assertEquals(emptyList<String>(), first.queries + second.queries)
Mockito.reset(repository)
Mockito.`when`(repository.insertIfAbsent(hash, "original source", "ko", "first", null, "v1")).thenThrow(failure)
assertSame(failure, assertThrows(IllegalStateException::class.java) { service.detectWithCache(query) })
assertEquals(listOf(query), first.queries)
assertEquals(emptyList<String>(), second.queries)
}
private fun service(providers: List<LanguageDetectionProvider>, order: List<String> = providers.map { it.providerName }) =
LanguageDetectionCacheService(repository, providers, LanguageProviderProperties(detectionOrder = order.joinToString(",")))
}
class RecordingDetectionProvider(
override val providerName: String,
private val response: () -> String?
) : LanguageDetectionProvider {
val queries = mutableListOf<String>()
override fun detect(query: String): String? {
queries.add(query)
return response()
}
}
@@ -0,0 +1,178 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.configs.QueryDslConfig
import kr.co.vividnext.sodalive.content.LanguageDetectionCacheService
import kr.co.vividnext.sodalive.content.LanguageDetectionResultRepository
import kr.co.vividnext.sodalive.content.theme.AudioContentThemeQueryRepository
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunity
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository
import kr.co.vividnext.sodalive.member.Member
import kr.co.vividnext.sodalive.member.MemberRepository
import kr.co.vividnext.sodalive.member.MemberRole
import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.adapter.out.persistence.CreatorCommunityTranslationRepository
import kr.co.vividnext.sodalive.v2.creator.channel.community.translation.application.CreatorCommunityTranslationService
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.mockito.Mockito
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest
import org.springframework.boot.test.context.TestConfiguration
import org.springframework.boot.test.mock.mockito.SpyBean
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Import
import org.springframework.transaction.PlatformTransactionManager
import org.springframework.transaction.annotation.Propagation
import org.springframework.transaction.annotation.Transactional
import org.springframework.transaction.support.TransactionTemplate
import java.time.LocalDateTime
import java.util.UUID
@DataJpaTest(
properties = [
"spring.cache.type=none",
"spring.datasource.url=jdbc:h2:mem:language-provider-cache;MODE=MySQL;NON_KEYWORDS=VALUE;DB_CLOSE_ON_EXIT=FALSE",
"sodalive.language-providers.detection-order=papago,secondary",
"sodalive.language-providers.translation-order=papago,secondary"
]
)
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
@Import(
QueryDslConfig::class,
AudioContentThemeQueryRepository::class,
LanguageProviderProperties::class,
LanguageDetectionCacheService::class,
TranslationCoordinator::class,
TranslationJobWorker::class,
TranslationReadModelMaterializer::class,
TranslationSourceExtractor::class,
TranslationJobScheduler::class,
ResourceTranslationJobScheduler::class,
CreatorCommunityTranslationService::class,
LanguageProviderCacheEndToEndTest.Providers::class
)
@Transactional(propagation = Propagation.NOT_SUPPORTED)
class LanguageProviderCacheEndToEndTest @Autowired constructor(
private val detector: LanguageDetectionCacheService,
private val detectionRepository: LanguageDetectionResultRepository,
private val memoryRepository: TranslationMemoryRepository,
private val scheduler: TranslationJobScheduler,
private val worker: TranslationJobWorker,
private val materializer: TranslationReadModelMaterializer,
private val displayService: CreatorCommunityTranslationService,
private val posts: CreatorCommunityRepository,
private val members: MemberRepository,
private val translations: CreatorCommunityTranslationRepository,
private val papagoDetection: RecordingDetectionProvider,
private val secondaryDetection: RecordingDetectionProvider,
private val papagoTranslation: RecordingTranslationProvider,
private val secondaryTranslation: RecordingTranslationProvider,
transactionManager: PlatformTransactionManager
) {
private val transaction = TransactionTemplate(transactionManager)
@SpyBean
private lateinit var jobRepository: TranslationJobRepository
@BeforeEach
fun useH2CompatibleClaimQuery() {
Mockito.doAnswer {
jobRepository.findAll().filter { it.status == TranslationJobStatus.PENDING }.mapNotNull { it.id }
}.`when`(jobRepository).findNextPendingGroupJobIdsForUpdate(Mockito.any(LocalDateTime::class.java) ?: LocalDateTime.now())
}
@Test
@DisplayName("보조 공급자의 감지를 H2에 귀속 저장하고 외부 호출 없이 캐시를 재사용한다")
fun shouldPersistSecondaryDetectionAndReuseCache() {
val source = "provider source ${UUID.randomUUID()}"
assertEquals(DetectedLanguage("ko", "secondary"), detector.detectWithCache(source))
assertEquals(DetectedLanguage("ko", "secondary"), detector.detectWithCache(source))
val detection = transaction.execute {
detectionRepository.findBySourceHashAndProviderAndNormalizationVersion(
SourceTextNormalizer.hash(source),
"secondary",
"v1"
)
}!!
assertEquals("secondary", detection.provider)
assertEquals("ko", detection.detectedLanguage)
assertEquals(1, papagoDetection.queries.size)
assertEquals(1, secondaryDetection.queries.size)
}
@Test
@DisplayName("보조 공급자의 번역을 H2에 귀속 저장하고 무호출 캐시와 실제 표시를 재사용한다")
fun shouldPersistSecondaryTranslationAndReuseCacheThroughWorkerAndMaterializer() {
val source = "translation source ${UUID.randomUUID()}"
val postId = persistPost(source)
schedule(postId, source)
assertTrue(worker.processNextGroup())
val memory = transaction.execute {
memoryRepository.findBySourceHashAndSourceLanguageAndTargetLanguageAndProviderAndNormalizationVersion(
SourceTextNormalizer.hash(source),
"ko",
"en",
"secondary",
"v1"
)
}
assertNotNull(memory)
assertEquals("secondary", memory!!.provider)
assertEquals("secondary-v1", memory.providerVersion)
assertEquals("secondary:0", memory.translatedText)
assertEquals(
TranslationJobStatus.COMPLETED,
transaction.execute { jobRepository.findAll().single { it.resourceId == postId }.status }
)
assertEquals("secondary:0", displayService.findDisplayContents(listOf(postId), "en")[postId]?.content)
transaction.executeWithoutResult {
translations.deleteAll()
jobRepository.deleteAll()
}
assertTrue(materializer.materialize(LanguageTranslationTargetType.CREATOR_COMMUNITY, postId, "en"))
schedule(postId, source)
assertTrue(worker.processNextGroup())
assertEquals(1, papagoTranslation.requests.size)
assertEquals(1, secondaryTranslation.requests.size)
assertEquals(listOf(source), secondaryTranslation.requests.single().texts)
assertEquals("secondary:0", displayService.findDisplayContents(listOf(postId), "en")[postId]?.content)
}
private fun schedule(postId: Long, source: String) {
scheduler.scheduleMissingTranslation(
LanguageTranslationTargetType.CREATOR_COMMUNITY,
postId,
"content",
source,
"ko",
"en"
)
}
private fun persistPost(source: String): Long = transaction.execute {
val member = members.save(
Member(password = "password", nickname = "provider-${UUID.randomUUID()}", role = MemberRole.CREATOR)
)
posts.save(CreatorCommunity(source, 0, true, false, languageCode = "ko").apply { this.member = member }).id!!
}!!
@TestConfiguration
class Providers {
@Bean
fun papagoDetection() = RecordingDetectionProvider("papago") { throw IllegalStateException("sensitive provider body") }
@Bean
fun secondaryDetection() = RecordingDetectionProvider("secondary") { "ko" }
@Bean
fun papagoTranslation() = RecordingTranslationProvider("papago") { TranslateResult(emptyList()) }
@Bean
fun secondaryTranslation() = RecordingTranslationProvider("secondary")
}
}
@@ -0,0 +1,100 @@
package kr.co.vividnext.sodalive.i18n.translation
import kr.co.vividnext.sodalive.content.LanguageDetectionCacheService
import kr.co.vividnext.sodalive.content.LanguageDetectionResultRepository
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.mockito.Mockito
import org.springframework.boot.test.context.TestConfiguration
import org.springframework.boot.test.context.runner.ApplicationContextRunner
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Import
class LanguageProviderSpringConfigurationTest {
private val runner = ApplicationContextRunner().withUserConfiguration(Providers::class.java)
@Test
@DisplayName("실제 Spring 설정 바인딩으로 감지만 순서를 바꾸어도 번역 순서는 유지한다")
fun shouldBindIndependentCapabilityOrders() {
listOf("papago,secondary" to "papago", "secondary,papago" to "secondary").forEach { (order, expected) ->
runner.withPropertyValues(
"sodalive.language-providers.detection-order=$order",
"sodalive.language-providers.translation-order=papago,secondary"
).run { context ->
assertNull(context.startupFailure)
val detector = context.getBean(LanguageDetectionCacheService::class.java)
assertEquals(expected, detector.detectWithCache("source")?.providerName)
assertEquals(
"papago",
context.getBean(TranslationCoordinator::class.java)
.translate(TranslateRequest(listOf("source"), "ko", "en")).providerName
)
val secondary = context.getBean("secondaryDetection", RecordingDetectionProvider::class.java)
assertEquals(if (expected == "secondary") 1 else 0, secondary.queries.size)
assertEquals(0, context.getBean("secondaryTranslation", RecordingTranslationProvider::class.java).requests.size)
}
}
runner.withPropertyValues("sodalive.language-providers.translation-order=secondary,papago").run { context ->
assertNull(context.startupFailure)
assertEquals(
"papago",
context.getBean(LanguageDetectionCacheService::class.java).detectWithCache("source")?.providerName
)
assertEquals(
"secondary",
context.getBean(TranslationCoordinator::class.java)
.translate(TranslateRequest(listOf("source"), "ko", "en")).providerName
)
}
}
@Test
@DisplayName("Spring 기본값은 두 기능 모두 파파고이며 잘못된 설정은 시작 시 실패한다")
fun shouldDefaultToPapagoAndRejectInvalidConfigurationAtStartup() {
runner.run { context ->
assertNull(context.startupFailure)
val properties = context.getBean(LanguageProviderProperties::class.java)
assertEquals(listOf("papago"), properties.detectionOrder)
assertEquals(listOf("papago"), properties.translationOrder)
}
listOf("detection-order", "translation-order").forEach { key ->
listOf("", "papago,papago", "unknown").forEach { value ->
runner.withPropertyValues("sodalive.language-providers.$key=$value").run { context ->
assertNotNull(context.startupFailure)
}
}
}
}
@TestConfiguration
@Import(LanguageProviderProperties::class, TranslationCoordinator::class, LanguageDetectionCacheService::class)
class Providers {
@Bean
fun memoryRepository(): TranslationMemoryRepository = Mockito.mock(TranslationMemoryRepository::class.java)
@Bean
fun detectionRepository(): LanguageDetectionResultRepository {
val repository = Mockito.mock(LanguageDetectionResultRepository::class.java)
listOf("papago", "secondary").forEach { provider ->
Mockito.`when`(repository.findDetectedLanguageForUpdate(SourceTextNormalizer.hash("source"), provider, "v1"))
.thenReturn("ko")
}
return repository
}
@Bean
fun papagoDetection() = RecordingDetectionProvider("papago") { "ko" }
@Bean
fun secondaryDetection() = RecordingDetectionProvider("secondary") { "ko" }
@Bean
fun papagoTranslation() = RecordingTranslationProvider("papago")
@Bean
fun secondaryTranslation() = RecordingTranslationProvider("secondary")
}
}
@@ -30,7 +30,7 @@ class TranslationJobWorkerTest {
val worker = TranslationJobWorker( val worker = TranslationJobWorker(
translationJobRepository = jobRepository, translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository, translationMemoryRepository = memoryRepository,
translationProvider = provider, translationCoordinator = TranslationCoordinator(listOf(provider), LanguageProviderProperties(), memoryRepository),
materializer = materializer, materializer = materializer,
transactionManager = TestTransactionManager() transactionManager = TestTransactionManager()
) )
@@ -54,7 +54,7 @@ class TranslationJobWorkerTest {
val worker = TranslationJobWorker( val worker = TranslationJobWorker(
translationJobRepository = jobRepository, translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository, translationMemoryRepository = memoryRepository,
translationProvider = provider, translationCoordinator = TranslationCoordinator(listOf(provider), LanguageProviderProperties(), memoryRepository),
materializer = materializer, materializer = materializer,
transactionManager = TestTransactionManager() transactionManager = TestTransactionManager()
) )
@@ -84,7 +84,7 @@ class TranslationJobWorkerTest {
val worker = TranslationJobWorker( val worker = TranslationJobWorker(
translationJobRepository = jobRepository, translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository, translationMemoryRepository = memoryRepository,
translationProvider = provider, translationCoordinator = TranslationCoordinator(listOf(provider), LanguageProviderProperties(), memoryRepository),
materializer = materializer, materializer = materializer,
transactionManager = TestTransactionManager() transactionManager = TestTransactionManager()
) )
@@ -129,7 +129,7 @@ class TranslationJobWorkerTest {
val worker = TranslationJobWorker( val worker = TranslationJobWorker(
translationJobRepository = jobRepository, translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository, translationMemoryRepository = memoryRepository,
translationProvider = provider, translationCoordinator = TranslationCoordinator(listOf(provider), LanguageProviderProperties(), memoryRepository),
materializer = materializer, materializer = materializer,
transactionManager = TestTransactionManager() transactionManager = TestTransactionManager()
) )
@@ -165,7 +165,7 @@ class TranslationJobWorkerTest {
val worker = TranslationJobWorker( val worker = TranslationJobWorker(
translationJobRepository = jobRepository, translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository, translationMemoryRepository = memoryRepository,
translationProvider = provider, translationCoordinator = TranslationCoordinator(listOf(provider), LanguageProviderProperties(), memoryRepository),
materializer = materializer, materializer = materializer,
transactionManager = TestTransactionManager() transactionManager = TestTransactionManager()
) )
@@ -198,7 +198,7 @@ class TranslationJobWorkerTest {
val worker = TranslationJobWorker( val worker = TranslationJobWorker(
translationJobRepository = jobRepository, translationJobRepository = jobRepository,
translationMemoryRepository = memoryRepository, translationMemoryRepository = memoryRepository,
translationProvider = provider, translationCoordinator = TranslationCoordinator(listOf(provider), LanguageProviderProperties(), memoryRepository),
materializer = materializer, materializer = materializer,
transactionManager = TestTransactionManager() transactionManager = TestTransactionManager()
) )
@@ -213,7 +213,7 @@ class TranslationJobWorkerTest {
assertEquals(TranslationJobStatus.PENDING, job.status) assertEquals(TranslationJobStatus.PENDING, job.status)
assertEquals(1, job.retryCount) assertEquals(1, job.retryCount)
assertEquals("provider down", job.lastErrorMessage) assertEquals("all translation providers failed", job.lastErrorMessage)
assertTrue(job.nextRetryAt.isAfter(beforeRetryAt)) assertTrue(job.nextRetryAt.isAfter(beforeRetryAt))
} }
@@ -19,9 +19,12 @@ import kr.co.vividnext.sodalive.content.theme.translation.ContentThemeTranslatio
import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository import kr.co.vividnext.sodalive.content.translation.ContentTranslationRepository
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunity import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunity
import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository import kr.co.vividnext.sodalive.explorer.profile.creatorCommunity.CreatorCommunityRepository
import kr.co.vividnext.sodalive.i18n.translation.LanguageProviderProperties
import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType import kr.co.vividnext.sodalive.i18n.translation.LanguageTranslationTargetType
import kr.co.vividnext.sodalive.i18n.translation.PapagoTranslationService
import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler import kr.co.vividnext.sodalive.i18n.translation.ResourceTranslationJobScheduler
import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer import kr.co.vividnext.sodalive.i18n.translation.SourceTextNormalizer
import kr.co.vividnext.sodalive.i18n.translation.TranslationCoordinator
import kr.co.vividnext.sodalive.i18n.translation.TranslationJob import kr.co.vividnext.sodalive.i18n.translation.TranslationJob
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository import kr.co.vividnext.sodalive.i18n.translation.TranslationJobRepository
import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler import kr.co.vividnext.sodalive.i18n.translation.TranslationJobScheduler
@@ -668,7 +671,11 @@ class CreatorCommunityTranslationServiceTest @Autowired constructor(
): TranslationReadModelMaterializer { ): TranslationReadModelMaterializer {
return TranslationReadModelMaterializer( return TranslationReadModelMaterializer(
sourceExtractor = sourceExtractor, sourceExtractor = sourceExtractor,
translationMemoryRepository = memoryRepository, translationCoordinator = TranslationCoordinator(
listOf(PapagoTranslationService("test-id", "test-secret")),
LanguageProviderProperties(),
memoryRepository
),
contentTranslationRepository = Mockito.mock(ContentTranslationRepository::class.java), contentTranslationRepository = Mockito.mock(ContentTranslationRepository::class.java),
aiCharacterTranslationRepository = Mockito.mock(AiCharacterTranslationRepository::class.java), aiCharacterTranslationRepository = Mockito.mock(AiCharacterTranslationRepository::class.java),
contentThemeTranslationRepository = Mockito.mock(ContentThemeTranslationRepository::class.java), contentThemeTranslationRepository = Mockito.mock(ContentThemeTranslationRepository::class.java),