Sharing a script to extract misspell words from other misspell dictionary for typos

[vikivivi]

After running the below script, you can get more misspell words not exist in typos dictionary.
I run the script on Ubuntu Linux 24.04.3 LTS

INFO: New misspell words found: 17836 OK, 1889 NOK
INFO: 17836 new misspell words created in "/tmp/dict-misspell/dictionary_new_OK.txt" but not available in "/tmp/dict-misspell/dictionary_crate_ci_typos.txt"

After running the script, you will see these files in directory /tmp/dict-misspell

dictionary_codespell.txt
dictionary_crate_ci_typos.txt
dictionary_new_NOK.txt
dictionary_new_OK_aoo_mozilla_en.txt
dictionary_new_OK.txt
wordlist_aoo_mozilla_en_GB.txt
wordlist_aoo_mozilla_en_US.txt

Script: tool-analysis-misspell-check-dict-for-more-misspell.sh

#!/bin/sh
# Author: vikivivi
# Version: 20250821
# Script: tool-analysis-misspell-check-dict-for-more-misspell.sh
# A script to extract new misspell words from codespell or crate-ci/typos.
# Perform spell check on misspell and suggestion words during extracting.
# Generated dictionary format is based on codespell format.

TEMP_DIR="/tmp/dict-misspell"

MISSPELL_PROJECT_OWNER="crate_ci_typos"
# MISSPELL_PROJECT_OWNER="codespell"

if [ "${MISSPELL_PROJECT_OWNER}" = "codespell" ]; then
    ALLOWED_CAPITALIZE_PROPER_NOUNS="yes"
    DICT_BASE_DICTIONARY="${TEMP_DIR}/dictionary_codespell.txt"
    DICT_BASE_MISSPELL="${TEMP_DIR}/dict_misspell_codespell.txt"

    DICT_SRC_DICTIONARY="${TEMP_DIR}/dictionary_crate_ci_typos.txt"
    DICT_SRC_MISSPELL="${TEMP_DIR}/dict_misspell_crate_ci_typos.txt"

elif [ "${MISSPELL_PROJECT_OWNER}" = "crate_ci_typos" ]; then
    ALLOWED_CAPITALIZE_PROPER_NOUNS="no"
    DICT_BASE_DICTIONARY="${TEMP_DIR}/dictionary_crate_ci_typos.txt"
    DICT_BASE_MISSPELL="${TEMP_DIR}/dict_misspell_crate_ci_typos.txt"

    DICT_SRC_DICTIONARY="${TEMP_DIR}/dictionary_codespell.txt"
    DICT_SRC_MISSPELL="${TEMP_DIR}/dict_misspell_codespell.txt"
fi

DICT_NEW_DICTIONARY_OK="${TEMP_DIR}/dictionary_new_OK.txt"
DICT_NEW_DICTIONARY_NOK="${TEMP_DIR}/dictionary_new_NOK.txt"
# https://www.grammarly.com/blog/punctuation-capitalization/capitalization-rules/
DICT_NEW_DICTIONARY_OK_PROPER_NOUNS="${TEMP_DIR}/dictionary_new_OK_proper_nouns.txt"
DICT_NEW_DICTIONARY_OK_AOO_MOZILLA_EN="${TEMP_DIR}/dictionary_new_OK_aoo_mozilla_en.txt"
DICT_NEW_MISSPELL="${TEMP_DIR}/dict_misspell_new.txt"

## Example: echo autodetect | hunspell -d en_GB,en_US -l
## Example: echo autodetectinvalid | hunspell -d en_GB,en_US -a
## Example: echo autodetect | aspell list --lang=en
## Example: echo autodetect | aspell --lang=en -a
SPELL_CHECKER="aspell"
SPELL_CHECKER_OPTIONS="list --lang=en"
SPELL_CHECK_WITH_AOO_MOZILLA_EN_DICT="yes"

func_software_installed() {
    SOFTWARE=$1

    INSTALLED=$(command -v "${SOFTWARE}")
    RETVAL=$?
    if [ "x${RETVAL}" != "x0" ] || [ "${INSTALLED}" = "" ]; then
        echo "ERROR: ${SOFTWARE} not installed"
    fi
}

func_dictionary_installed() {
    DICTIONARY_FILE=$1
    DEB_FILE=$2

    if [ ! -f "${DICTIONARY_FILE}" ]; then
        echo "ERROR: Dictionary ${DICTIONARY_FILE} not installed. ${DEB_FILE}"
    fi
}

func_spelling_tools_installed_check() {
    # sudo apt install hunspell hunspell-en-gb hunspell-en-us
    func_software_installed "hunspell"
    func_dictionary_installed "/usr/share/hunspell/en_GB.dic"  "hunspell-en-gb"
    func_dictionary_installed "/usr/share/hunspell/en_US.dic"  "hunspell-en-us"

    # sudo apt install aspell aspell-en
    func_software_installed "aspell"
    func_dictionary_installed "/usr/share/aspell/en-common.cwl.gz" "aspell-en"

    func_software_install "curl"
}

func_get_marcoagpinto_aoo_mozilla_en_dict() {
    mkdir -p "${TEMP_DIR}"
    if [ "${SPELL_CHECK_WITH_AOO_MOZILLA_EN_DICT}" = "yes" ]; then
        curl "https://raw.githubusercontent.com/marcoagpinto/aoo-mozilla-en-dict/refs/heads/master/en_GB%20(Marco%20Pinto)%20(-ise)%20(2025%2B)/wordlist_marcoagpinto_20250701_275092w_ise.txt" -o "${TEMP_DIR}/wordlist_aoo_mozilla_en_GB.txt"
        RETVAL=$?
        if [ "${RETVAL}" != "0" ]; then
            echo "ERROR: Failed to download aoo-mozilla-en-dict"
            exit 1
        fi

        curl "https://raw.githubusercontent.com/marcoagpinto/aoo-mozilla-en-dict/refs/heads/master/en_US%20(Kevin%20Atkinson)/wordlist_kevin_en_US_20201207_123679w.txt" -o "${TEMP_DIR}/wordlist_aoo_mozilla_en_US.txt"
        RETVAL=$?
        if [ "${RETVAL}" != "0" ]; then
            echo "ERROR: Failed to download aoo-mozilla-en-dict"
            exit 1
        fi
    fi
}

func_get_crate_ci_typos_dict() {
    mkdir -p "${TEMP_DIR}"
    curl "https://raw.githubusercontent.com/crate-ci/typos/refs/heads/master/crates/typos-dict/assets/words.csv" -o "${TEMP_DIR}/words.csv"
    RETVAL=$?
    if [ "${RETVAL}" != "0" ]; then
        echo "ERROR: Failed to download crate-ci/typos dictionary"
        exit 1
    fi

    sed -i 's/,/->/' "${TEMP_DIR}/words.csv"
    sed -i 's/,/, /g' "${TEMP_DIR}/words.csv"
    sed -i '/, / s/$/,/' "${TEMP_DIR}/words.csv"
    mv -f "${TEMP_DIR}/words.csv" "${TEMP_DIR}/dictionary_crate_ci_typos.txt"
}

func_get_codespell_dict() {
    mkdir -p "${TEMP_DIR}"
    curl "https://raw.githubusercontent.com/codespell-project/codespell/refs/heads/main/codespell_lib/data/dictionary.txt" -o "${TEMP_DIR}/dictionary.txt"
    RETVAL=$?
    if [ "${RETVAL}" != "0" ]; then
        echo "ERROR: Failed to download codespell dictionary"
        exit 1
    fi

    mv -f "${TEMP_DIR}/dictionary.txt" "${TEMP_DIR}/dictionary_codespell.txt"
}

func_extract_misspell_words() {
    FILE_IN=$1
    FILE_OUT=$2

    if [ "${FILE_IN}" = "" ]; then
        echo "ERROR: FILE_IN not defined"
        exit 1
    fi
    if [ "${FILE_OUT}" = "" ]; then
        echo "ERROR: FILE_OUT not defined"
        exit 1
    fi
    if [ ! -f "${FILE_IN}" ]; then
        echo "ERROR: FILE_IN not exist"
        exit 1
    fi

    awk -F"->" '{print $1}' "${FILE_IN}" > "${FILE_OUT}"
}

func_extract_new_misspell_words() {
    # Extract words present in DICT_SRC_MISSPELL but not in DICT_BASE_MISSPELL
    grep -vxF -f "${DICT_BASE_MISSPELL}" "${DICT_SRC_MISSPELL}" > "${DICT_NEW_MISSPELL}"
}

func_extract_new_misspell_words_and_suggestion() {
    rm -f "${DICT_NEW_DICTIONARY_OK}"
    rm -f "${DICT_NEW_DICTIONARY_NOK}"
    rm -f "${DICT_NEW_DICTIONARY_OK_PROPER_NOUNS}"
    rm -f "${DICT_NEW_DICTIONARY_OK_AOO_MOZILLA_EN}"

    grep -v '^ *#' < "${DICT_NEW_MISSPELL}" | while IFS= read -r MISSPELL_WORD; do
        MISSPELL_SUGGESTION_STRING=$(grep "^${MISSPELL_WORD}->" "${DICT_SRC_DICTIONARY}")
        if [ "${MISSPELL_SUGGESTION_STRING}" != "" ]; then
            # Checking misspell word is not a valid word in spell checker
            MISSPELL_WORDS_CHECKED=$(echo "${MISSPELL_WORD}" | "${SPELL_CHECKER}" ${SPELL_CHECKER_OPTIONS})
            SUGGESTION_WORDS=$(echo "${MISSPELL_SUGGESTION_STRING}" | awk -F"->" '{print $2}')
            if [ "${SUGGESTION_WORDS}" != "" ]; then
                # Checking suggested words is a valid word in spell checker
                SUGGESTION_WORDS_CHECKED=$(echo "${SUGGESTION_WORDS}" | "${SPELL_CHECKER}" ${SPELL_CHECKER_OPTIONS})
                if [ "${SUGGESTION_WORDS_CHECKED}" != "" ] && [ "${MISSPELL_WORDS_CHECKED}" != "" ]; then
                    # Failed Spell Checker: Suggested word: Failed (Not a valid word), Misspell word: Passed (Not a valid word)
                    if [ "${SPELL_CHECK_WITH_AOO_MOZILLA_EN_DICT}" = "yes" ]; then
                        # Try searching suggested word in aoo_mozilla dictionaries if allowed
                        AOO_MOZILLA_WORDS_CHECKED=$(grep "^${SUGGESTION_WORDS}$" "${TEMP_DIR}/wordlist_aoo_mozilla_en_"*.txt | awk -F":" '{ print $2}' | head -n 1)
                        if [ "${AOO_MOZILLA_WORDS_CHECKED}" != "" ]; then
                            echo "${MISSPELL_SUGGESTION_STRING}" >> "${DICT_NEW_DICTIONARY_OK_AOO_MOZILLA_EN}"
                        fi
                    fi
                    if [ "${ALLOWED_CAPITALIZE_PROPER_NOUNS}" = "yes" ]; then
                        # Failed Spell Checker, try capitalize as proper nouns if allowed
                        SUGGESTION_PROPER_NOUNS_WORDS=$(echo "${MISSPELL_SUGGESTION_STRING}" | awk -F"->" '{print $2}' | sed 's/\b\(.\)/\U\1/g')
                        SUGGESTION_PROPER_NOUNS_WORDS_CHECKED=$(echo "${SUGGESTION_PROPER_NOUNS_WORDS}" | "${SPELL_CHECKER}" ${SPELL_CHECKER_OPTIONS})
                        if [ "${SUGGESTION_PROPER_NOUNS_WORDS_CHECKED}" = "" ]; then
                            echo "${MISSPELL_WORD}->${SUGGESTION_PROPER_NOUNS_WORDS}"
                            echo "${MISSPELL_WORD}->${SUGGESTION_PROPER_NOUNS_WORDS}" >> "${DICT_NEW_DICTIONARY_OK_PROPER_NOUNS}"
                        fi
                    fi
                fi
                if [ "${SUGGESTION_WORDS_CHECKED}" = "" ] && [ "${MISSPELL_WORDS_CHECKED}" != "" ]; then
                    # Passed Spell Checker with both misspell and suggested words
                    echo "${MISSPELL_SUGGESTION_STRING}"
                    echo "${MISSPELL_SUGGESTION_STRING}" >> "${DICT_NEW_DICTIONARY_OK}"
                else
                    # Failed Spell Checker or Failed Proper Noun
                    echo "# ${MISSPELL_SUGGESTION_STRING}"
                    echo "${MISSPELL_SUGGESTION_STRING}" >> "${DICT_NEW_DICTIONARY_NOK}"
                fi
            fi
        fi
    done

    MISSPELL_NEW_COUNT_OK=$(wc -l "${DICT_NEW_DICTIONARY_OK}" | awk '{print $1}')
    MISSPELL_NEW_COUNT_NOK=$(wc -l "${DICT_NEW_DICTIONARY_NOK}" | awk '{print $1}')
    echo ""
    echo "INFO: New misspell words found: ${MISSPELL_NEW_COUNT_OK} OK, ${MISSPELL_NEW_COUNT_NOK} NOK"
    echo "INFO: ${MISSPELL_NEW_COUNT_OK} new misspell words created in \"${DICT_NEW_DICTIONARY_OK}\" but not available in \"${DICT_BASE_DICTIONARY}\""
    if [ "${ALLOWED_CAPITALIZE_PROPER_NOUNS}" = "yes" ]; then
        MISSPELL_NEW_COUNT_OK_PROPER_NOUNS=$(wc -l "${DICT_NEW_DICTIONARY_OK_PROPER_NOUNS}" | awk '{print $1}')
        echo "INFO: ${MISSPELL_NEW_COUNT_OK_PROPER_NOUNS} new misspell words created in \"${DICT_NEW_DICTIONARY_OK_PROPER_NOUNS}\" but not available in \"${DICT_BASE_DICTIONARY}\""
    fi
    if [ "${SPELL_CHECK_WITH_AOO_MOZILLA_EN_DICT}" = "yes" ]; then
        MISSPELL_NEW_COUNT_OK_AOO_MOZILLA_EN_DICT=$(wc -l "${DICT_NEW_DICTIONARY_OK_AOO_MOZILLA_EN}" | awk '{print $1}')
        echo "INFO: ${MISSPELL_NEW_COUNT_OK_AOO_MOZILLA_EN_DICT} new misspell words created in \"${DICT_NEW_DICTIONARY_OK_AOO_MOZILLA_EN}\" but not available in \"${DICT_BASE_DICTIONARY}\""
    fi

    rm -f "${DICT_BASE_MISSPELL}"
    rm -f "${DICT_SRC_MISSPELL}"
    rm -f "${DICT_NEW_MISSPELL}"
}

func_spelling_tools_installed_check
func_get_marcoagpinto_aoo_mozilla_en_dict
func_get_crate_ci_typos_dict
func_get_codespell_dict
func_extract_misspell_words "${DICT_BASE_DICTIONARY}" "${DICT_BASE_MISSPELL}"
func_extract_misspell_words "${DICT_SRC_DICTIONARY}" "${DICT_SRC_MISSPELL}"
func_extract_new_misspell_words
func_extract_new_misspell_words_and_suggestion