diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java index 27e1bc36..cbf4e98e 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilder.java @@ -2,11 +2,31 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import org.springframework.web.util.HtmlUtils; import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; @Component public class FewShotSearchTextBuilder { + static final int MAX_IDENTIFIER_LENGTH = 100; + static final int MAX_SHORT_FIELD_LENGTH = 200; + static final int MAX_REQUIREMENTS_LENGTH = 1_200; + static final int MAX_QUESTION_LENGTH = 600; + static final int MAX_ANSWER_LENGTH = 2_000; + static final int MAX_TAGS_LENGTH = 500; + + private static final Pattern SCRIPT_STYLE_PATTERN = Pattern.compile( + "(?is)<(script|style)\\b[^>]*>.*?" + ); + private static final Pattern BLOCK_HTML_TAG_PATTERN = Pattern.compile( + "(?is)]*>" + ); + private static final Pattern HTML_COMMENT_PATTERN = Pattern.compile("(?s)"); + private static final Pattern HTML_TAG_PATTERN = Pattern.compile("(?is)]*>"); + private static final Pattern WHITESPACE_PATTERN = Pattern.compile("[\\p{Z}\\s]+"); + public String buildQueryText(FewShotSearchQuery query) { return """ [JOB_CATEGORY] @@ -27,12 +47,12 @@ public String buildQueryText(FewShotSearchQuery query) { [ANSWER] %s """.formatted( - value(query.jobCategory()), - value(query.jobTitle()), - lines(query.mainTasks()), - lines(query.qualifications()), - value(query.question()), - value(query.answer()) + value(query.jobCategory(), MAX_SHORT_FIELD_LENGTH), + value(query.jobTitle(), MAX_SHORT_FIELD_LENGTH), + lines(query.mainTasks(), MAX_REQUIREMENTS_LENGTH), + lines(query.qualifications(), MAX_REQUIREMENTS_LENGTH), + value(query.question(), MAX_QUESTION_LENGTH), + value(query.answer(), MAX_ANSWER_LENGTH) ); } @@ -63,31 +83,63 @@ public String buildCandidateDocument(FewShotCase fewShotCase) { [TAGS] %s """.formatted( - value(fewShotCase.id()), + value(fewShotCase.id(), MAX_IDENTIFIER_LENGTH), fewShotCase.source(), - value(fewShotCase.jobCategory()), - value(fewShotCase.jobTitle()), - lines(fewShotCase.mainTasks()), - lines(fewShotCase.qualifications()), - value(fewShotCase.question()), - value(fewShotCase.sanitizedAnswer()), - String.join(", ", fewShotCase.tags()) + value(fewShotCase.jobCategory(), MAX_SHORT_FIELD_LENGTH), + value(fewShotCase.jobTitle(), MAX_SHORT_FIELD_LENGTH), + lines(fewShotCase.mainTasks(), MAX_REQUIREMENTS_LENGTH), + lines(fewShotCase.qualifications(), MAX_REQUIREMENTS_LENGTH), + value(fewShotCase.question(), MAX_QUESTION_LENGTH), + value(fewShotCase.sanitizedAnswer(), MAX_ANSWER_LENGTH), + values(fewShotCase.tags(), MAX_TAGS_LENGTH) ); } - private static String lines(List values) { + private static String lines(List values, int maxLength) { if (values == null || values.isEmpty()) { return ""; } - return values.stream() + String joined = values.stream() + .filter(StringUtils::hasText) + .map(FewShotSearchTextBuilder::normalize) .filter(StringUtils::hasText) - .map(String::trim) .map(line -> line.startsWith("-") ? line : "- " + line) - .reduce((left, right) -> left + "\n" + right) - .orElse(""); + .collect(Collectors.joining("\n")); + return truncate(joined, maxLength); + } + + private static String values(List values, int maxLength) { + if (values == null || values.isEmpty()) { + return ""; + } + String joined = values.stream() + .filter(StringUtils::hasText) + .map(FewShotSearchTextBuilder::normalize) + .filter(StringUtils::hasText) + .collect(Collectors.joining(", ")); + return truncate(joined, maxLength); + } + + private static String value(String value, int maxLength) { + return truncate(normalize(value), maxLength); + } + + private static String normalize(String value) { + if (!StringUtils.hasText(value)) { + return ""; + } + String decoded = HtmlUtils.htmlUnescape(value); + String withoutExecutableContent = SCRIPT_STYLE_PATTERN.matcher(decoded).replaceAll(" "); + String withoutComments = HTML_COMMENT_PATTERN.matcher(withoutExecutableContent).replaceAll(" "); + String withBlockSeparators = BLOCK_HTML_TAG_PATTERN.matcher(withoutComments).replaceAll(" "); + String withoutTags = HTML_TAG_PATTERN.matcher(withBlockSeparators).replaceAll(""); + return WHITESPACE_PATTERN.matcher(withoutTags).replaceAll(" ").trim(); } - private static String value(String value) { - return StringUtils.hasText(value) ? value.trim() : ""; + private static String truncate(String value, int maxLength) { + if (value.length() <= maxLength) { + return value; + } + return value.substring(0, maxLength).trim(); } } diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java index 189683b1..ee03128b 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/service/ai/fewshot/FewShotSearchTextBuilderTest.java @@ -62,4 +62,50 @@ void buildsCandidateDocument() { .contains("Spring Boot 경험") .doesNotContain("embedding"); } + + @Test + @DisplayName("HTML을 제거하고 반복 공백을 정규화한 뒤 검색 텍스트를 구성한다") + void normalizesHtmlAndWhitespace() { + String text = builder.buildQueryText(new FewShotSearchQuery( + "EV-01", + " 백엔드 개발 ", + "Backend Engineer", + List.of("

Spring Boot

API 개발"), + List.of("Java\n\t개발 경험"), + "지원 직무 경험", + "JPA로 API를 개발했습니다." + )); + + assertThat(text) + .contains("[JOB_CATEGORY]\n백엔드 개발") + .contains("[JOB_TITLE]\nBackend Engineer") + .contains("[MAIN_TASKS]\n- Spring Boot API 개발") + .contains("[QUALIFICATIONS]\n- Java 개발 경험") + .contains("[QUESTION]\n지원 직무 경험") + .contains("[ANSWER]\nJPA로 API를 개발했습니다.") + .doesNotContain("", "