[Fix] Few-shot query embedding 캐시 추가

src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/DefaultFewShotSearchService.java

@@ -20,11 +20,18 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Pattern;
 
 @Service
 @Slf4j
 public class DefaultFewShotSearchService implements FewShotSearchService {
+    private static final long QUERY_EMBEDDING_CACHE_CLEANUP_INTERVAL_MILLIS = 60_000L;
+    private static final long DEFAULT_QUERY_EMBEDDING_IN_FLIGHT_WAIT_TIMEOUT_MILLIS = 20_000L;
     private static final Pattern TOKEN_SPLIT_PATTERN = Pattern.compile("[^\\p{IsAlphabetic}\\p{IsDigit}가-힣]+");
     private static final Pattern NORMALIZED_INPUT_WHITESPACE_PATTERN = Pattern.compile("[\\p{Z}\\s]+");
 
@@ -33,6 +40,11 @@ public class DefaultFewShotSearchService implements FewShotSearchService {
     private final CohereEmbeddingClient cohereEmbeddingClient;
     private final FewShotProperties properties;
     private final Map<String, SelectionCacheEntry> selectionCache = new ConcurrentHashMap<>();
+    private final Map<String, QueryEmbeddingCacheEntry> queryEmbeddingCache = new ConcurrentHashMap<>();
+    private final Map<String, CompletableFuture<QueryEmbeddingCacheEntry>> queryEmbeddingInFlight =
+            new ConcurrentHashMap<>();
+    private final AtomicLong queryEmbeddingCacheNextCleanupAt = new AtomicLong();
+    private final AtomicBoolean queryEmbeddingCacheCleanupInProgress = new AtomicBoolean();
     private final Map<String, DocumentEmbeddingCacheEntry> documentEmbeddingCache = new ConcurrentHashMap<>();
     private final Map<String, CompletableFuture<DocumentEmbeddingCacheEntry>> documentEmbeddingInFlight =
             new ConcurrentHashMap<>();
@@ -115,7 +127,7 @@ private List<SelectedFewShotCase> selectWithCohere(
             List<String> documents = candidates.stream()
                     .map(textBuilder::buildCandidateDocument)
                     .toList();
-            float[] queryEmbedding = cohereEmbeddingClient.embedQuery(queryText);
+            float[] queryEmbedding = resolveQueryEmbedding(queryText);
             List<float[]> documentEmbeddings = resolveDocumentEmbeddings(candidates, documents);
             List<SelectedFewShotCase> ranked = new ArrayList<>();
             List<Double> similarityScores = new ArrayList<>();
@@ -161,6 +173,118 @@ private static String formatScore(double score) {
         return String.format(Locale.ROOT, "%.4f", score);
     }
 
+    private float[] resolveQueryEmbedding(String queryText) {
+        if (!properties.isCacheEnabled()) {
+            return cohereEmbeddingClient.embedQuery(queryText);
+        }
+        Instant now = Instant.now();
+        maintainQueryEmbeddingCache(now);
+        String key = sha256(queryText);
+        QueryEmbeddingCacheEntry cached = readQueryEmbeddingCache(key, now);
+        if (cached != null) {
+            log.debug("few-shot query embedding cache hit.");
+            return cached.embedding();
+        }
+
+        CompletableFuture<QueryEmbeddingCacheEntry> created = new CompletableFuture<>();
+        CompletableFuture<QueryEmbeddingCacheEntry> existing = queryEmbeddingInFlight.putIfAbsent(key, created);
+        if (existing != null) {
+            log.debug("few-shot query embedding in-flight request reused.");
+            return awaitQueryEmbedding(existing).embedding();
+        }
+
+        QueryEmbeddingCacheEntry cachedAfterClaim = readQueryEmbeddingCache(key, Instant.now());
+        if (cachedAfterClaim != null) {
+            created.complete(cachedAfterClaim);
+            queryEmbeddingInFlight.remove(key, created);
+            log.debug("few-shot query embedding cache hit after in-flight claim.");
+            return cachedAfterClaim.embedding();
+        }
+
+        try {
+            QueryEmbeddingCacheEntry initialized = new QueryEmbeddingCacheEntry(
+                    cohereEmbeddingClient.embedQuery(queryText),
+                    expiresAt()
+            );
+            queryEmbeddingCache.put(key, initialized);
+            maintainQueryEmbeddingCache(Instant.now());
+            created.complete(initialized);
+            log.debug("few-shot query embedding cache initialized.");
+            return initialized.embedding();
+        } catch (RuntimeException | Error e) {
+            created.completeExceptionally(e);
+            throw e;
+        } finally {
+            queryEmbeddingInFlight.remove(key, created);
+        }
+    }
+
+    private QueryEmbeddingCacheEntry readQueryEmbeddingCache(String key, Instant now) {
+        QueryEmbeddingCacheEntry cached = queryEmbeddingCache.get(key);
+        if (cached == null) {
+            return null;
+        }
+        if (cached.expiresAt().isBefore(now)) {
+            queryEmbeddingCache.remove(key, cached);
+            return null;
+        }
+        QueryEmbeddingCacheEntry accessed = cached.accessedAt(now);
+        queryEmbeddingCache.replace(key, cached, accessed);
+        return accessed;
+    }
+
+    private QueryEmbeddingCacheEntry awaitQueryEmbedding(
+            CompletableFuture<QueryEmbeddingCacheEntry> existing
+    ) {
+        long timeoutMillis = properties.getQueryEmbeddingInFlightWaitTimeout() == null
+                ? DEFAULT_QUERY_EMBEDDING_IN_FLIGHT_WAIT_TIMEOUT_MILLIS
+                : Math.max(1L, properties.getQueryEmbeddingInFlightWaitTimeout().toMillis());
+        try {
+            return existing.get(timeoutMillis, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("Few-shot query embedding 대기 중 인터럽트되었습니다.", e);
+        } catch (ExecutionException e) {
+            throw new IllegalStateException("공유된 Few-shot query embedding 생성에 실패했습니다.", e.getCause());
+        } catch (TimeoutException e) {
+            throw new IllegalStateException("공유된 Few-shot query embedding 대기 시간이 초과되었습니다.", e);
+        }
+    }
+
+    private void maintainQueryEmbeddingCache(Instant now) {
+        int maxSize = Math.max(1, properties.getQueryEmbeddingCacheMaxSize());
+        long nowMillis = now.toEpochMilli();
+        if (queryEmbeddingCache.size() < maxSize
+                && nowMillis < queryEmbeddingCacheNextCleanupAt.get()) {
+            return;
+        }
+        if (!queryEmbeddingCacheCleanupInProgress.compareAndSet(false, true)) {
+            return;
+        }
+        try {
+            int sizeBefore = queryEmbeddingCache.size();
+            queryEmbeddingCache.entrySet().removeIf(entry -> entry.getValue().expiresAt().isBefore(now));
+            int sizeAfterExpiration = queryEmbeddingCache.size();
+            if (sizeAfterExpiration >= maxSize) {
+                int trimTarget = Math.max(1, maxSize - Math.max(1, maxSize / 10));
+                int removalCount = sizeAfterExpiration - trimTarget;
+                queryEmbeddingCache.entrySet().stream()
+                        .sorted(Comparator.comparing(entry -> entry.getValue().lastAccessedAt()))
+                        .limit(removalCount)
+                        .forEach(entry -> queryEmbeddingCache.remove(entry.getKey(), entry.getValue()));
+            }
+            queryEmbeddingCacheNextCleanupAt.set(nowMillis + QUERY_EMBEDDING_CACHE_CLEANUP_INTERVAL_MILLIS);
+            log.debug(
+                    "few-shot query embedding cache maintained. sizeBefore={}, sizeAfter={}, maxSize={}",
+                    sizeBefore,
+                    queryEmbeddingCache.size(),
+                    maxSize
+            );
+        } finally {
+            queryEmbeddingCacheCleanupInProgress.set(false);
+        }
+    }
+
     private List<float[]> resolveDocumentEmbeddings(
             List<FewShotCase> candidates,
             List<String> documents
@@ -512,4 +636,23 @@ public float[] embedding() {
             return embedding.clone();
         }
     }
+
+    private record QueryEmbeddingCacheEntry(float[] embedding, Instant expiresAt, Instant lastAccessedAt) {
+        private QueryEmbeddingCacheEntry(float[] embedding, Instant expiresAt) {
+            this(embedding, expiresAt, Instant.now());
+        }
+
+        private QueryEmbeddingCacheEntry {
+            embedding = embedding == null ? new float[0] : embedding.clone();
+        }
+
+        private QueryEmbeddingCacheEntry accessedAt(Instant accessedAt) {
+            return new QueryEmbeddingCacheEntry(embedding, expiresAt, accessedAt);
+        }
+
+        @Override
+        public float[] embedding() {
+            return embedding.clone();
+        }
+    }
 }

src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotProperties.java

@@ -16,6 +16,8 @@ public class FewShotProperties {
     private boolean fallbackEnabled = true;
     private boolean cacheEnabled = true;
     private Duration cacheTtl = Duration.ofMinutes(30);
+    private int queryEmbeddingCacheMaxSize = 1_000;
+    private Duration queryEmbeddingInFlightWaitTimeout = Duration.ofSeconds(20);
     private Source source = new Source();
     private Search search = new Search();
 
@@ -83,6 +85,22 @@ public void setCacheTtl(Duration cacheTtl) {
         this.cacheTtl = cacheTtl;
     }
 
+    public int getQueryEmbeddingCacheMaxSize() {
+        return queryEmbeddingCacheMaxSize;
+    }
+
+    public void setQueryEmbeddingCacheMaxSize(int queryEmbeddingCacheMaxSize) {
+        this.queryEmbeddingCacheMaxSize = queryEmbeddingCacheMaxSize;
+    }
+
+    public Duration getQueryEmbeddingInFlightWaitTimeout() {
+        return queryEmbeddingInFlightWaitTimeout;
+    }
+
+    public void setQueryEmbeddingInFlightWaitTimeout(Duration queryEmbeddingInFlightWaitTimeout) {
+        this.queryEmbeddingInFlightWaitTimeout = queryEmbeddingInFlightWaitTimeout;
+    }
+
     public Source getSource() {
         return source;
     }

src/main/java/com/jobdri/jobdri_api/global/cohere/CohereEmbeddingClient.java

@@ -94,7 +94,7 @@ private CohereEmbeddingResponse callCohere(CohereEmbeddingRequest request) {
                 if (attempt == MAX_TRANSIENT_ATTEMPTS) {
                     throw unavailable("Cohere Embed API가 일시적으로 응답할 수 없습니다.", e);
                 }
-                Duration delay = e.retryAfter() != null ? e.retryAfter() : backoff;
+                Duration delay = boundedRetryDelay(e.retryAfter(), backoff);
                 log.warn(
                         "Cohere Embed API transient failure. attempt={}, maxAttempts={}, retryAfterMs={}, message={}",
                         attempt,
@@ -260,6 +260,11 @@ private static Duration nextBackoff(Duration current) {
         return next.compareTo(MAX_RETRY_BACKOFF) > 0 ? MAX_RETRY_BACKOFF : next;
     }
 
+    static Duration boundedRetryDelay(Duration retryAfter, Duration backoff) {
+        Duration requested = retryAfter != null ? retryAfter : backoff;
+        return requested.compareTo(MAX_RETRY_BACKOFF) > 0 ? MAX_RETRY_BACKOFF : requested;
+    }
+
     private static void sleepBeforeRetry(Duration delay) {
         try {
             Thread.sleep(delay.toMillis());

src/main/resources/application-analysis-eval.yaml

@@ -95,6 +95,8 @@ analysis:
     fallback-enabled: ${ANALYSIS_FEW_SHOT_FALLBACK_ENABLED:true}
     cache-enabled: ${ANALYSIS_FEW_SHOT_CACHE_ENABLED:true}
     cache-ttl: ${ANALYSIS_FEW_SHOT_CACHE_TTL:30m}
+    query-embedding-cache-max-size: ${ANALYSIS_FEW_SHOT_QUERY_EMBEDDING_CACHE_MAX_SIZE:1000}
+    query-embedding-in-flight-wait-timeout: ${ANALYSIS_FEW_SHOT_QUERY_EMBEDDING_IN_FLIGHT_WAIT_TIMEOUT:20s}
     source:
       fixed-enabled: ${ANALYSIS_FEW_SHOT_FIXED_ENABLED:true}
       curated-enabled: ${ANALYSIS_FEW_SHOT_CURATED_ENABLED:true}

src/main/resources/application-dev.yaml

@@ -169,6 +169,8 @@ analysis:
     fallback-enabled: ${ANALYSIS_FEW_SHOT_FALLBACK_ENABLED:true}
     cache-enabled: ${ANALYSIS_FEW_SHOT_CACHE_ENABLED:true}
     cache-ttl: ${ANALYSIS_FEW_SHOT_CACHE_TTL:30m}
+    query-embedding-cache-max-size: ${ANALYSIS_FEW_SHOT_QUERY_EMBEDDING_CACHE_MAX_SIZE:1000}
+    query-embedding-in-flight-wait-timeout: ${ANALYSIS_FEW_SHOT_QUERY_EMBEDDING_IN_FLIGHT_WAIT_TIMEOUT:20s}
     source:
       fixed-enabled: ${ANALYSIS_FEW_SHOT_FIXED_ENABLED:true}
       curated-enabled: ${ANALYSIS_FEW_SHOT_CURATED_ENABLED:true}

src/main/resources/application-prod.yaml

@@ -171,6 +171,8 @@ analysis:
     fallback-enabled: ${ANALYSIS_FEW_SHOT_FALLBACK_ENABLED:true}
     cache-enabled: ${ANALYSIS_FEW_SHOT_CACHE_ENABLED:true}
     cache-ttl: ${ANALYSIS_FEW_SHOT_CACHE_TTL:30m}
+    query-embedding-cache-max-size: ${ANALYSIS_FEW_SHOT_QUERY_EMBEDDING_CACHE_MAX_SIZE:1000}
+    query-embedding-in-flight-wait-timeout: ${ANALYSIS_FEW_SHOT_QUERY_EMBEDDING_IN_FLIGHT_WAIT_TIMEOUT:20s}
     source:
       fixed-enabled: ${ANALYSIS_FEW_SHOT_FIXED_ENABLED:true}
       curated-enabled: ${ANALYSIS_FEW_SHOT_CURATED_ENABLED:true}