diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchService.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchService.java index b303a702..032cd3bc 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchService.java @@ -76,7 +76,11 @@ public List searchRelevantFewShots(FewShotSearchQuery query FewShotSelectionMode selectionMode = selected.isEmpty() ? FewShotSelectionMode.STATIC_FALLBACK : FewShotSelectionMode.EMBEDDING; - if (selected.isEmpty() && properties.isFallbackEnabled()) { + int minimumSelectedCount = Math.max( + 1, + Math.min(properties.getSearch().getMinimumSelectedCount(), requestedTopK) + ); + if (selected.size() < minimumSelectedCount && properties.isFallbackEnabled()) { selected = selectLocally(query, candidates, requestedTopK, "local-fallback"); if (!selected.isEmpty()) { selectionMode = FewShotSelectionMode.LOCAL_FALLBACK; @@ -114,12 +118,15 @@ private List selectWithCohere( float[] queryEmbedding = cohereEmbeddingClient.embedQuery(queryText); List documentEmbeddings = resolveDocumentEmbeddings(candidates, documents); List ranked = new ArrayList<>(); + List similarityScores = new ArrayList<>(); for (int i = 0; i < candidates.size(); i++) { double score = cosineSimilarity(queryEmbedding, documentEmbeddings.get(i)); - if (score >= properties.getSearch().getMinRerankScore()) { + similarityScores.add(score); + if (score >= properties.getSearch().getMinSimilarity()) { ranked.add(new SelectedFewShotCase(candidates.get(i), score, "cohere-embedding")); } } + logSimilarityDistribution(similarityScores, ranked.size()); ranked.sort(Comparator .comparingDouble(SelectedFewShotCase::score).reversed() .thenComparingInt(item -> -item.fewShotCase().priority()) @@ -132,6 +139,28 @@ private List selectWithCohere( } } + private void logSimilarityDistribution(List scores, int passedThresholdCount) { + if (scores.isEmpty()) { + return; + } + double topScore = scores.stream().mapToDouble(Double::doubleValue).max().orElse(0.0); + double bottomScore = scores.stream().mapToDouble(Double::doubleValue).min().orElse(0.0); + double avgScore = scores.stream().mapToDouble(Double::doubleValue).average().orElse(0.0); + log.info( + "few-shot embedding similarity distribution. candidateCount={}, passedThresholdCount={}, minSimilarity={}, topScore={}, bottomScore={}, avgScore={}", + scores.size(), + passedThresholdCount, + formatScore(properties.getSearch().getMinSimilarity()), + formatScore(topScore), + formatScore(bottomScore), + formatScore(avgScore) + ); + } + + private static String formatScore(double score) { + return String.format(Locale.ROOT, "%.4f", score); + } + private List resolveDocumentEmbeddings( List candidates, List documents @@ -314,6 +343,10 @@ private String selectionCacheKey(FewShotSearchQuery query, int topK, String data return sha256( datasetFingerprint + "\n" + topK + + "\n" + properties.getSearch().getMinSimilarity() + + "\n" + properties.getSearch().getMinimumSelectedCount() + + "\n" + properties.isFallbackEnabled() + + "\n" + properties.getSearch().isDiversityEnabled() + "\n" + defaultString(query.caseId()) + "\n" + textBuilder.buildQueryText(query) ); diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotProperties.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotProperties.java index dd6857db..7d883fdd 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotProperties.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotProperties.java @@ -141,7 +141,8 @@ public void setReviewedProductionEnabled(boolean reviewedProductionEnabled) { public static class Search { private int candidateLimit = 30; private int topK = 5; - private double minRerankScore = -1.0; + private double minSimilarity = -1.0; + private int minimumSelectedCount = 1; private boolean diversityEnabled = true; public int getCandidateLimit() { @@ -161,11 +162,27 @@ public void setTopK(int topK) { } public double getMinRerankScore() { - return minRerankScore; + return minSimilarity; } public void setMinRerankScore(double minRerankScore) { - this.minRerankScore = minRerankScore; + this.minSimilarity = minRerankScore; + } + + public double getMinSimilarity() { + return minSimilarity; + } + + public void setMinSimilarity(double minSimilarity) { + this.minSimilarity = minSimilarity; + } + + public int getMinimumSelectedCount() { + return minimumSelectedCount; + } + + public void setMinimumSelectedCount(int minimumSelectedCount) { + this.minimumSelectedCount = minimumSelectedCount; } public boolean isDiversityEnabled() { diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProvider.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProvider.java index 4a8aaec1..f814eae0 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProvider.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProvider.java @@ -114,6 +114,8 @@ private Map fewShotPolicy() { fewShotPolicy.put("datasetVersion", fewShotProperties.getDatasetVersion()); fewShotPolicy.put("topK", fewShotProperties.getSearch().getTopK()); fewShotPolicy.put("candidateLimit", fewShotProperties.getSearch().getCandidateLimit()); + fewShotPolicy.put("minSimilarity", fewShotProperties.getSearch().getMinSimilarity()); + fewShotPolicy.put("minimumSelectedCount", fewShotProperties.getSearch().getMinimumSelectedCount()); fewShotPolicy.put("reviewedEvaluationEnabled", fewShotProperties.getSource().isReviewedEvaluationEnabled()); return fewShotPolicy; } diff --git a/src/main/resources/application-analysis-eval.yaml b/src/main/resources/application-analysis-eval.yaml index 6da33b6c..24f93dc9 100644 --- a/src/main/resources/application-analysis-eval.yaml +++ b/src/main/resources/application-analysis-eval.yaml @@ -103,7 +103,8 @@ analysis: search: candidate-limit: ${ANALYSIS_FEW_SHOT_CANDIDATE_LIMIT:30} top-k: ${ANALYSIS_FEW_SHOT_TOP_K:5} - min-rerank-score: ${ANALYSIS_FEW_SHOT_MIN_RERANK_SCORE:-1.0} + min-similarity: ${ANALYSIS_FEW_SHOT_MIN_SIMILARITY:${ANALYSIS_FEW_SHOT_MIN_RERANK_SCORE:-1.0}} + minimum-selected-count: ${ANALYSIS_FEW_SHOT_MINIMUM_SELECTED_COUNT:1} diversity-enabled: ${ANALYSIS_FEW_SHOT_DIVERSITY_ENABLED:true} cohere: diff --git a/src/main/resources/application-dev.yaml b/src/main/resources/application-dev.yaml index 7505ae0b..2f32e674 100644 --- a/src/main/resources/application-dev.yaml +++ b/src/main/resources/application-dev.yaml @@ -177,7 +177,8 @@ analysis: search: candidate-limit: ${ANALYSIS_FEW_SHOT_CANDIDATE_LIMIT:30} top-k: ${ANALYSIS_FEW_SHOT_TOP_K:5} - min-rerank-score: ${ANALYSIS_FEW_SHOT_MIN_RERANK_SCORE:-1.0} + min-similarity: ${ANALYSIS_FEW_SHOT_MIN_SIMILARITY:${ANALYSIS_FEW_SHOT_MIN_RERANK_SCORE:-1.0}} + minimum-selected-count: ${ANALYSIS_FEW_SHOT_MINIMUM_SELECTED_COUNT:1} diversity-enabled: ${ANALYSIS_FEW_SHOT_DIVERSITY_ENABLED:true} cohere: diff --git a/src/main/resources/application-prod.yaml b/src/main/resources/application-prod.yaml index b351123f..4426a5f2 100644 --- a/src/main/resources/application-prod.yaml +++ b/src/main/resources/application-prod.yaml @@ -179,7 +179,8 @@ analysis: search: candidate-limit: ${ANALYSIS_FEW_SHOT_CANDIDATE_LIMIT:30} top-k: ${ANALYSIS_FEW_SHOT_TOP_K:5} - min-rerank-score: ${ANALYSIS_FEW_SHOT_MIN_RERANK_SCORE:-1.0} + min-similarity: ${ANALYSIS_FEW_SHOT_MIN_SIMILARITY:${ANALYSIS_FEW_SHOT_MIN_RERANK_SCORE:-1.0}} + minimum-selected-count: ${ANALYSIS_FEW_SHOT_MINIMUM_SELECTED_COUNT:1} diversity-enabled: ${ANALYSIS_FEW_SHOT_DIVERSITY_ENABLED:true} cohere: diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchServiceTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchServiceTest.java index 56759859..8f9b9968 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchServiceTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchServiceTest.java @@ -97,6 +97,43 @@ void fallsBackToLocalSelectionWhenCohereFails() { assertThat(result.getFirst().selectionMethod()).isEqualTo("local-fallback"); } + @Test + @DisplayName("임계값을 통과한 후보가 최소 개수보다 적으면 로컬 선택으로 fallback한다") + void fallsBackToLocalSelectionWhenEmbeddingResultsAreBelowMinimumCount() { + properties.setDynamicSelectionEnabled(true); + properties.getSearch().setMinSimilarity(0.5); + properties.getSearch().setMinimumSelectedCount(2); + when(caseStore.loadActiveCases()).thenReturn(List.of( + caseItem("FS-1", "Spring Boot API 개발", 0), + caseItem("FS-2", "브랜드 운영", 0) + )); + when(cohereEmbeddingClient.embedQuery(any())).thenReturn(new float[]{1, 0}); + when(cohereEmbeddingClient.embedDocuments(any())).thenReturn(List.of( + new float[]{1, 0}, + new float[]{0, 1} + )); + + List result = service.searchRelevantFewShots(query("EV-99"), 2); + + assertThat(result).hasSize(2); + assertThat(result).allMatch(item -> item.selectionMethod().equals("local-fallback")); + } + + @Test + @DisplayName("fallback이 비활성화되면 최소 유사도 미만 후보를 선택하지 않는다") + void excludesCandidatesBelowMinimumSimilarityWhenFallbackDisabled() { + properties.setDynamicSelectionEnabled(true); + properties.setFallbackEnabled(false); + properties.getSearch().setMinSimilarity(0.5); + when(caseStore.loadActiveCases()).thenReturn(List.of(caseItem("FS-1", "브랜드 운영", 0))); + when(cohereEmbeddingClient.embedQuery(any())).thenReturn(new float[]{1, 0}); + when(cohereEmbeddingClient.embedDocuments(any())).thenReturn(List.of(new float[]{0, 1})); + + List result = service.searchRelevantFewShots(query("EV-99"), 1); + + assertThat(result).isEmpty(); + } + @Test @DisplayName("서로 다른 검색 요청에서도 동일한 후보의 document embedding을 재사용한다") void reusesDocumentEmbeddingAcrossDifferentQueries() { diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProviderTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProviderTest.java index d36a882a..c17cb584 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProviderTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/core/AnalysisInputFingerprintProviderTest.java @@ -100,6 +100,34 @@ void fingerprintIgnoresCorpusQuestionDistance() { assertThat(provider.create(first)).isEqualTo(provider.create(changedDistance)); } + @Test + @DisplayName("Few-shot 유사도 임계값이 달라지면 fingerprint가 달라진다") + void fingerprintChangesWhenFewShotSimilarityPolicyChanges() { + when(fewShotPromptProvider.getPrompt()).thenReturn("few-shot"); + FewShotProperties defaultProperties = new FewShotProperties(); + FewShotProperties changedProperties = new FewShotProperties(); + changedProperties.getSearch().setMinSimilarity(0.5); + changedProperties.getSearch().setMinimumSelectedCount(2); + AnalysisExecutionPayload payload = payload(currentJobPosting(), similarContext("Spring Boot API 개발", 0.91)); + + assertThat(providerWith(defaultProperties).create(payload)) + .isNotEqualTo(providerWith(changedProperties).create(payload)); + } + + private AnalysisInputFingerprintProvider providerWith(FewShotProperties properties) { + return new AnalysisInputFingerprintProvider( + new ObjectMapper(), + fewShotPromptProvider, + properties, + new CohereProperties(null, null, null), + "gpt-4o-mini", + false, + "", + 3, + 5 + ); + } + private AnalysisExecutionPayload payload(JobPosting jobPosting, SimilarJobPostingContext context) { return new AnalysisExecutionPayload( 1L,