ollama_semantic_function_calling / ollamatools-new / keet / node_modules / @holepunchto / keet-translations / check-consistency.js

import { translations } from './index.js'

const REFERENCE_LOCALE = 'en'

const referenceSegments = getSegments(REFERENCE_LOCALE)
Object.freeze(referenceSegments)
export const locales = Object.keys(translations)
Object.freeze(locales)

export function checkReferenceQuality () {
  const untrimmedSegs = getSpacyOverview(referenceSegments)
  const duplicatesOverview = getDuplicatesOverview(referenceSegments)

  const res = []
  if (untrimmedSegs.length > 0) {
    res.push(`There are ${untrimmedSegs.length} segments with surrounding space`)
    untrimmedSegs.map((seg) => res.push(`    "${seg}"`))
  }

  if (duplicatesOverview.size > 0) {
    res.push(
      `The reference (${REFERENCE_LOCALE}) contains ${duplicatesOverview.size}` +
        ' duplicate entries'
    )
    for (const [normSeg, segs] of duplicatesOverview) {
      res.push(`    "${segs.join('" and "')}"     => all normalise to "${normSeg}"`)
    }
  }

  return res
}

export function checkTranslationErrors () {
  const summary = []
  const details = []

  for (const loc of locales) {
    checkAllSegmentsTranslated(loc, summary, details)
    checkNoExtraSegments(loc, summary, details)
    checkNoEmptySegments(loc, summary, details)
  }

  return { summary, details }
}

export function checkTranslationWarnings () {
  const summary = []
  const details = []

  for (const loc of locales) {
    checkConsistentPunctuation(loc, summary, details)

    const untrimmedSegs = getSpacyOverview(getSegments(loc))
    if (untrimmedSegs.length > 0) {
      const msg = `${loc} has ${untrimmedSegs.length} segments with surrounding space`
      summary.push(msg)
      details.push(msg)
      untrimmedSegs.map((seg) => details.push(`"${seg}"`))
    }
  }

  return { summary, details }
}

function getDuplicatesOverview (segments) {
  const normalise = (seg) => seg.toLowerCase().trim()

  const overview = new Map()
  for (const seg of segments) {
    const normSeg = normalise(seg)

    const value = (overview.get(normSeg) || [])
    value.push(seg)

    overview.set(normSeg, value)
  }

  const res = Array.from(overview).filter(([normSeg, segs]) => segs.length > 1)
  return new Map(res)
}

function getSpacyOverview (segments) {
  return segments.filter((seg) => seg.trim() !== seg)
}

export function getMissingSegments (loc) {
  const translatedSegments = getSegments(loc)
  const missingSegments = referenceSegments.filter(
    (seg) => !translatedSegments.includes(seg)
  )

  return missingSegments
}

function checkAllSegmentsTranslated (loc, summary, details) {
  const missingSegments = getMissingSegments(loc)

  if (missingSegments.length > 0) {
    const msg = `${loc} has ${missingSegments.length} missing segments`
    details.push(msg)
    summary.push(msg)
    missingSegments.map((seg) => details.push(`\t'${seg}'`))
  }
}

function checkNoExtraSegments (loc, summary, details) {
  const translatedSegments = getSegments(loc)
  const extraSegments = translatedSegments.filter(
    (seg) => !referenceSegments.includes(seg)
  )

  if (extraSegments.length > 0) {
    const msg = `${loc} has ${extraSegments.length} extra segments`
    summary.push(msg)
    details.push(msg)
    extraSegments.map((seg) => details.push(`\t'${seg}'`))
  }
}

function checkNoEmptySegments (loc, summary, details) {
  const entries = getEntries(loc)
  const segments = getSegments(loc)

  const emptyStrRegex = /^\s*$/
  const emptySegments = segments.filter((seg) => emptyStrRegex.test(entries[seg]))
  if (emptySegments.length > 0) {
    const msg = `${loc} has ${emptySegments.length} empty segments`
    summary.push(msg)
    details.push(msg)
    emptySegments.map((seg) => details.push(`\t'${seg}'`))
  }
}

// Note: this is only relevant for languages with punctuation similar to English,
// not for languages like Chinese or Japanese
function checkConsistentPunctuation (loc, summary, details) {
  const ignore = ['hi-IN', 'ka-GE', 'zh-CN', 'zh-TW']
  if (ignore.includes(loc)) {
    return
  }

  const entries = getEntries(loc)

  const endsOnPunctRegex = /.*[.,#!$%;:=()]$/
  const flaggedSegments = referenceSegments.filter(
    (seg) => endsOnPunctRegex.test(seg)
      // Note: no-entry case is not flagged here (flagged elsewhere)
      ? entries[seg] && seg.slice(-1) !== entries[seg].slice(-1)
      : false
  )
  if (flaggedSegments.length > 0) {
    const msg = `${loc} has ${flaggedSegments.length} segments with inconsistent` +
      ' ending punctuation compared to source'

    summary.push(msg)
    details.push(msg)
    flaggedSegments.map((seg) => details.push(`\t'${seg}' <-> '${entries[seg]}'`))
  }
}

export function getEntries (locale) {
  return translations[locale].translations
}

function getSegments (locale) {
  return Object.keys(getEntries(locale))
}