Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions scripts/check-library-content.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ beforeEach(() => {
'https://www.sim.ai/models/anthropic',
'https://www.sim.ai/models/anthropic/claude-opus-4-6',
]),
simCatalog: { integrations: 280, actions: 5826, triggers: 538 },
}
mkdirSync(path.join(config.contentDir, 'authors'), { recursive: true })
writeFileSync(path.join(config.contentDir, 'authors', 'sim.json'), '{"id":"sim","name":"Sim"}')
Expand Down Expand Up @@ -318,6 +319,61 @@ describe('check-library-content', () => {
expect(findings[0]).toMatchObject({ line: 12, rule: 'mdx' })
})

it.each([
['Sim connects 1,000+ integrations and every major LLM.', '1,000+ integrations', 'overstates'],
[
"Sim's [100+ integrations](https://www.sim.ai/integrations) cover the basics.",
'100+ integrations',
'stale',
],
['Sim ships 9,000+ actions across its catalog.', '9,000+ actions', 'overstates'],
['Sim listens for events with 300+ triggers.', '300+ triggers', 'stale'],
['Sim has **1,000+** integrations.', '1,000+ integrations', 'overstates'],
['Sim has 1,000+ Integrations.', '1,000+ Integrations', 'overstates'],
])('rejects a Sim count the catalog does not support: %s', async (body, claim, verdict) => {
writePost('library', 'post', body)
expect(await findingsFor('library', 'post')).toEqual([
{
line: 10,
rule: 'sim-count',
message: expect.stringMatching(new RegExp(`${claim.replace('+', '\\+')}.*${verdict}`)),
},
])
})

it('rejects an unsupported Sim count in a frontmatter FAQ answer, at its line', async () => {
writePost('library', 'post', 'Body.', {
faq: '\n - q: "How many integrations does Sim have?"\n a: "Sim has 1,000+ integrations."',
})
expect(await findingsFor('library', 'post')).toEqual([
expect.objectContaining({ line: 10, rule: 'sim-count' }),
])
})

it('accepts true Sim counts and leaves competitor counts alone', async () => {
writePost(
'library',
'post',
[
'Sim has 280+ integrations, 5,800+ actions, and 530+ triggers.',
"Sim's pricing page documents more than 270 integrations.",
'Sim fits teams that want a collaborative visual builder, 270+ integrations, and an Apache 2.0 core.',
"Zapier lists 9,000+ integrations, while Sim's 280+ integrations focus on agents.",
'Compared with Sim, MindStudio advertises 1,000+ integrations.',
'Sim is open source. Workato cites 14,000+ integrations.',
'Sim can call any tool that the connector marketplaces built on open protocols expose, often 1,000+ integrations.',
'```text\nSim has 1,000+ integrations\n```',
'````md\n```text\nSim has 1,000+ integrations\n````',
].join('\n\n')
)
expect(await findingsFor('library', 'post')).toEqual([])
})

it('does not check dated blog or changelog posts, whose counts were true when published', async () => {
writePost('blog', 'v0-5', 'Sim v0.5 adds 100+ integrations spanning 300+ actions.')
expect(await findingsFor('blog', 'v0-5')).toEqual([])
})

it('rejects an FAQ heading in the body', async () => {
writePost('library', 'post', 'Intro.\n\n## FAQs\n\nQ and A.')
expect(await findingsFor('library', 'post')).toEqual([
Expand Down
73 changes: 71 additions & 2 deletions scripts/check-library-content.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,22 @@
* - `search-title` / `search-description`: a published post's search title (`seoTitle ?? title`
* with the " | Sim" suffix) fits in 60 characters and its search description
* (`seoDescription ?? description`) in 160, the lengths Google shows before truncating.
* - `sim-count`: a library post's count of Sim's integrations, actions, or triggers is true
* against the generated catalog (`INTEGRATION_TOTALS`): no higher than the total, and no
* lower than 90% of it. Answer engines quote these figures, so an overclaim such as
* "1,000+ integrations" or a figure left from an older catalog spreads to third-party sites.
* A count belongs to Sim when the nearest "Sim" before it on the line is at most 12 words back
* with no capitalized word between, which leaves competitor counts alone.
* Blog and changelog posts are dated, so their counts are true as of publication.
*
* Run one post with `--slug <section>/<slug>` or `--slug <slug>`.
*/
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
import path from 'node:path'
import { compile } from '@mdx-js/mdx'
import { INTEGRATION_TOTALS } from '@sim/deployment-config/integration-totals'
import integrationsJson from '@sim/deployment-config/integrations.json'
import { formatLowerBound } from '@sim/utils/formatting'
import { SIM_DOCS_URL, SIM_SITE_URL } from '@sim/utils/site'
import matter from 'gray-matter'
import type { Root } from 'mdast'
Expand Down Expand Up @@ -65,6 +74,10 @@ export type Rule =
| 'internal-link'
| 'search-title'
| 'search-description'
| 'sim-count'

/** The catalog units a Sim count claim can name. */
type SimCountUnit = 'integrations' | 'actions' | 'triggers'

/** Appended to every post title by the root layout's `title.template`. */
const SEARCH_TITLE_SUFFIX = ' | Sim'
Expand Down Expand Up @@ -94,6 +107,8 @@ export interface ContentCheckConfig {
customerSlugs: readonly string[]
/** Canonical integration, model, and documentation pages served by the current source tree. */
linkedPages: ReadonlySet<string>
/** Sim's catalog totals that `sim-count` holds library claims to. */
simCatalog: Readonly<Record<SimCountUnit, number>>
}

/** Post folders per section, keyed by slug, with each post's `draft` flag. */
Expand All @@ -116,8 +131,43 @@ const TRAILING_PUNCTUATION = /[.,;:!]+$/
const LINK_TARGET_OPENER = /(?:\]\(\s*|href=\{?["'`])$/
const MARKDOWN_LINK = /\[[^\]\n]*\]\([^)\n]*\)/
const FAQ_HEADING = /^#{1,6}\s+FAQs?\s*:?\s*$/i
const CODE_FENCE = /^\s*(```|~~~)/
/** An opening or closing code fence: three or more backticks or tildes. */
const CODE_FENCE = /^\s*(`{3,}|~{3,})/
const ROUTE_FILES = ['page.tsx', 'page.ts', 'route.tsx', 'route.ts']
/** A catalog count such as "1,000+ integrations": group 1 is the number, group 2 the unit. */
const COUNT_CLAIM = /\b(\d{1,3}(?:,\d{3})+|\d+)\+?\s+(integrations|actions|triggers)\b/gi
/** Inline emphasis and code markers, which would otherwise split "**1,000+** integrations". */
const INLINE_MARKERS = /\*\*|__|[*`]/g
const SIM_NAME = /\bSim\b/g
const SIM_CLAIM_MAX_WORDS = 12
/** A Sim count below this share of the catalog total is left from an older catalog. */
const SIM_COUNT_FLOOR = 0.9

/**
* The counts on one line of text that belong to Sim, with the claim as written. A sentence or
* clause that moves on to another product starts with a capitalized word, which ends Sim's claim.
*/
function simCountClaims(line: string): { claim: string; count: number; unit: SimCountUnit }[] {
const claims: { claim: string; count: number; unit: SimCountUnit }[] = []
const text = line.replace(INLINE_MARKERS, '')
for (const match of text.matchAll(COUNT_CLAIM)) {
const before = text.slice(0, match.index)
const lastSim = [...before.matchAll(SIM_NAME)].at(-1)
if (lastSim?.index === undefined) continue
const words = before
.slice(lastSim.index + lastSim[0].length)
.split(/\s+/)
.map((word) => word.replace(/^[^A-Za-z0-9]+/, ''))
.filter(Boolean)
if (words.length > SIM_CLAIM_MAX_WORDS || words.some((word) => /^[A-Z]/.test(word))) continue
claims.push({
claim: match[0],
count: Number.parseInt(match[1].replaceAll(',', ''), 10),
unit: match[2].toLowerCase() as SimCountUnit,
})
}
return claims
}

function isSection(value: string): value is Section {
return (SECTIONS as readonly string[]).includes(value)
Expand Down Expand Up @@ -397,6 +447,23 @@ export async function checkPost(

const data = parsed.data as Record<string, unknown>

const checkSimCounts = (text: string, line: number) => {
if (section !== 'library') return
for (const { claim, count, unit } of simCountClaims(text)) {
const total = config.simCatalog[unit]
if (count <= total && count >= total * SIM_COUNT_FLOOR) continue
report(
line,
'sim-count',
count > total
? `"${claim}" overstates Sim's catalog, which has ${total} ${unit}.`
: `"${claim}" is stale: Sim's catalog has ${total} ${unit}.`,
`Write ${formatLowerBound(total)} ${unit}, the catalog-derived figure (INTEGRATION_TOTALS in packages/deployment-config/src/integration-totals.ts).`
)
}
}
frontmatterLines.forEach((text, index) => checkSimCounts(text, index + 2))

if (section === 'changelog' && result.success) {
const { date, updated, draft, ogAlt } = result.data
if (!draft && (date.getTime() > Date.now() || (updated && updated.getTime() > Date.now()))) {
Expand Down Expand Up @@ -535,11 +602,12 @@ export async function checkPost(
const fenceMatch = CODE_FENCE.exec(text)
if (fenceMatch) {
if (fence === null) fence = fenceMatch[1]
else if (fenceMatch[1] === fence) fence = null
else if (fenceMatch[1][0] === fence[0] && fenceMatch[1].length >= fence.length) fence = null
return
}
if (fence !== null) return
const line = bodyOffset + index + 1
checkSimCounts(text, line)
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
if (FAQ_HEADING.test(text.trim())) {
report(
line,
Expand Down Expand Up @@ -771,6 +839,7 @@ async function main() {
movedBlogSlugs: LIBRARY_MOVED_BLOG_SLUGS,
customerSlugs: CUSTOMER_STORIES.map((story) => story.slug),
linkedPages: readLinkedPages(path.join(root, 'apps/docs')),
simCatalog: INTEGRATION_TOTALS,
}

const args = process.argv.slice(2)
Expand Down