initial commit

FunASR/fun_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py

+import os
+
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.date import DateFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.fraction import FractionFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.punctuation import PunctuationFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.time import TimeFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.whitelist import WhiteListFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.word import WordFst
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+    generator_main,
+)
+from pynini.lib import pynutil
+
+import logging
+
+
+class ClassifyFst(GraphFst):
+    """
+    Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+
+    Args:
+        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
+        overwrite_cache: set to True to overwrite .far files
+    """
+
+    def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
+        super().__init__(name="tokenize_and_classify", kind="classify")
+
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            far_file = os.path.join(cache_dir, "_ko_itn.far")
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
+            logging.info(f"ClassifyFst.fst was restored from {far_file}.")
+        else:
+            logging.info(f"Creating ClassifyFst grammars.")
+            cardinal = CardinalFst()
+            cardinal_graph = cardinal.fst
+
+            decimal = DecimalFst(cardinal)
+            decimal_graph = decimal.fst
+
+            fraction = FractionFst(cardinal)
+            fraction_graph = fraction.fst
+
+            measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst
+            date_graph = DateFst().fst
+            word_graph = WordFst().fst
+            time_graph = TimeFst().fst
+            money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst
+            whitelist_graph = WhiteListFst().fst
+            punct_graph = PunctuationFst().fst
+            electronic_graph = ElectronicFst().fst
+            telephone_graph = TelephoneFst(cardinal).fst
+
+            classify = (
+                pynutil.add_weight(whitelist_graph, 1.01)
+                | pynutil.add_weight(time_graph, 1.1)
+                | pynutil.add_weight(date_graph, 1.09)
+                | pynutil.add_weight(decimal_graph, 1.1)
+                | pynutil.add_weight(fraction_graph, 1.1)
+                | pynutil.add_weight(measure_graph, 1.1)
+                | pynutil.add_weight(cardinal_graph, 1.1)
+                | pynutil.add_weight(money_graph, 1.1)
+                | pynutil.add_weight(telephone_graph, 1.1)
+                | pynutil.add_weight(electronic_graph, 1.1)
+                | pynutil.add_weight(word_graph, 100)
+            )
+
+            punct = (
+                pynutil.insert("tokens { ")
+                + pynutil.add_weight(punct_graph, weight=1.1)
+                + pynutil.insert(" }")
+            )
+            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
+            token_plus_punct = (
+                pynini.closure(punct + pynutil.insert(" "))
+                + token
+                + pynini.closure(pynutil.insert(" ") + punct)
+            )
+
+            graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
+            graph = delete_space + graph + delete_space
+
+            self.fst = graph.optimize()
+
+            if far_file:
+                generator_main(far_file, {"tokenize_and_classify": self.fst})
+                logging.info(f"ClassifyFst grammars are saved to {far_file}.")

FunASR/fun_text_processing/inverse_text_normalization/ko/taggers/whitelist.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, convert_space
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for classifying whitelisted tokens
+        e.g. misses -> tokens { name: "mrs." }
+    This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
+    """
+
+    def __init__(self):
+        super().__init__(name="whitelist", kind="classify")
+
+        whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert()
+        graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"')
+        self.fst = graph.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/taggers/word.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_SPACE, GraphFst
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class.
+        e.g. sleep -> tokens { name: "sleep" }
+    """
+
+    def __init__(self):
+        super().__init__(name="word", kind="classify")
+        word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
+        self.fst = word.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/utils.py

+import csv
+import os
+from typing import Union
+
+import inflect
+
+_inflect = inflect.engine()
+
+
+def num_to_word(x: Union[str, int]):
+    """
+    converts integer to spoken representation
+    Args
+        x: integer
+    Returns: spoken representation
+    """
+    if isinstance(x, int):
+        x = str(x)
+        x = _inflect.number_to_words(str(x)).replace("-", " ").replace(",", "")
+    return x
+
+
+def get_abs_path(rel_path):
+    """
+    Get absolute path
+
+    Args:
+        rel_path: relative path to this file
+
+    Returns absolute path
+    """
+    return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path
+
+
+def load_labels(abs_path):
+    """
+    loads relative path file as dictionary
+
+    Args:
+        abs_path: absolute path
+
+    Returns dictionary of mappings
+    """
+    label_tsv = open(abs_path, encoding="utf-8")
+    labels = list(csv.reader(label_tsv, delimiter="\t"))
+    return labels
+
+
+def augment_labels_with_punct_at_end(labels):
+    """
+    augments labels: if key ends on a punctuation that value does not have, add a new label
+    where the value maintains the punctuation
+
+    Args:
+        labels : input labels
+    Returns:
+        additional labels
+    """
+    res = []
+    for label in labels:
+        if len(label) > 1:
+            if label[0][-1] == "." and label[1][-1] != ".":
+                res.append([label[0], label[1] + "."] + label[2:])
+    return res

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing cardinal
+        e.g. cardinal { integer: "23" negative: "-" } -> -23
+    """
+
+    def __init__(self):
+        super().__init__(name="cardinal", kind="verbalize")
+        optional_sign = pynini.closure(
+            pynutil.delete("negative:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space,
+            0,
+            1,
+        )
+        graph = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        self.numbers = graph
+        graph = optional_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/date.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for verbalizing date, e.g.
+        date { month: "january" day: "5" year: "2012" preserve_order: true } -> february 5 2012
+        date { day: "5" month: "january" year: "2012" preserve_order: true } -> 5 february 2012
+    """
+
+    def __init__(self):
+        super().__init__(name="date", kind="verbalize")
+        month = (
+            pynutil.delete("month:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynutil.insert(" ")
+        )
+        day = (
+            pynutil.delete("day:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynutil.insert(" ")
+        )
+        year = (
+            pynutil.delete("year:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynutil.insert(" ")
+        )
+
+        # month (day) year
+        graph_mdy = (
+            month
+            + pynini.closure(delete_extra_space + day, 0, 1)
+            + pynini.closure(delete_extra_space + year, 0, 1)
+        )
+
+        # (day) month year
+        graph_dmy = (
+            pynini.closure(day + delete_extra_space, 0, 1)
+            + month
+            + pynini.closure(delete_extra_space + year, 0, 1)
+        )
+
+        optional_preserve_order = pynini.closure(
+            pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
+            | pynutil.delete("field_order:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space
+        )
+
+        # year month day
+        graph_ymd = year + month + day
+
+        # month day
+        graph_md = month + day
+
+        # year month
+        graph_ym = year + month
+
+        # add some grammars
+        final_graph = (
+            (graph_mdy | year | graph_dmy | graph_ymd | graph_md | graph_ym | month | day)
+            + delete_space
+            + optional_preserve_order
+        )
+
+        delete_tokens = self.delete_tokens(final_graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing decimal, e.g.
+        decimal { negative: "true" integer_part: "12"  fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
+    """
+
+    def __init__(self):
+        super().__init__(name="decimal", kind="verbalize")
+        optionl_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        integer = (
+            pynutil.delete("integer_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_integer = pynini.closure(integer + delete_space, 0, 1)
+        fractional = (
+            pynutil.insert(".")
+            + pynutil.delete("fractional_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
+        quantity = (
+            pynutil.delete("quantity:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1)
+        graph = optional_integer + optional_fractional + optional_quantity
+        self.numbers = graph
+        graph = optionl_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/electronic.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu
+    """
+
+    def __init__(self):
+        super().__init__(name="electronic", kind="verbalize")
+        user_name = (
+            pynutil.delete("username:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        domain = (
+            pynutil.delete("domain:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        protocol = (
+            pynutil.delete("protocol:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = user_name + delete_space + pynutil.insert("@") + domain
+        graph |= protocol
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py

+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+
+import pynini
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for verbalizing fraction,
+    """
+
+    def __init__(self):
+        super().__init__(name="fraction", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        numerator = (
+            pynutil.delete("numerator:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        denominator = (
+            pynutil.delete("denominator:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = (
+            optional_sign + numerator + delete_space + pynutil.insert("/") + denominator
+        ).optimize()
+        self.numbers = graph
+        delete_tokens = self.delete_tokens(optional_sign + graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/measure.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure, e.g.
+        measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg
+
+    Args:
+        decimal: DecimalFst
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, decimal: GraphFst, cardinal: GraphFst):
+        super().__init__(name="measure", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
+        unit = (
+            pynutil.delete("units:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+            + delete_space
+        )
+        graph_decimal = (
+            pynutil.delete("decimal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + decimal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph_cardinal = (
+            pynutil.delete("cardinal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + cardinal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert(" ") + unit
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/money.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing money, e.g.
+        money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05
+
+    Args:
+        decimal: DecimalFst
+    """
+
+    def __init__(self, decimal: GraphFst):
+        super().__init__(name="money", kind="verbalize")
+        unit = (
+            pynutil.delete("currency:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = unit + delete_space + decimal.numbers
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing ordinal, e.g.
+       ordinal { integer: "13" } -> 13th
+    """
+
+    def __init__(self):
+        super().__init__(name="ordinal", kind="verbalize")
+        graph = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        convert_eleven = pynini.cross("11", "11th")
+        convert_twelve = pynini.cross("12", "12th")
+        convert_thirteen = pynini.cross("13", "13th")
+        convert_one = pynini.cross("1", "1st")
+        convert_two = pynini.cross("2", "2nd")
+        convert_three = pynini.cross("3", "3rd")
+        convert_rest = pynutil.insert("th", weight=0.01)
+
+        suffix = pynini.cdrewrite(
+            convert_eleven
+            | convert_twelve
+            | convert_thirteen
+            | convert_one
+            | convert_two
+            | convert_three
+            | convert_rest,
+            "",
+            "[EOS]",
+            DAMO_SIGMA,
+        )
+        graph = graph @ suffix
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for verbalizing telephone, e.g.
+        telephone { number_part: "123-123-5678" }
+        -> 123-123-5678
+    """
+
+    def __init__(self):
+        super().__init__(name="telephone", kind="verbalize")
+
+        number_part = (
+            pynutil.delete('number_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_country_code = pynini.closure(
+            pynutil.delete('country_code: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynini.accep(" "),
+            0,
+            1,
+        )
+        delete_tokens = self.delete_tokens(optional_country_code + number_part)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/time.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    DAMO_DIGIT,
+    GraphFst,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing time, e.g.
+        time { hours: "12" minutes: "30" } -> 12:30
+        time { hours: "1" minutes: "12" } -> 01:12
+        time { hours: "2" suffix: "a.m." } -> 02:00 a.m.
+    """
+
+    def __init__(self):
+        super().__init__(name="time", kind="verbalize")
+        add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
+            pynutil.insert("0") + DAMO_DIGIT
+        )
+        # hour
+        hour = (
+            pynutil.delete("hours:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+        # minute
+        minute = (
+            pynutil.delete("minutes:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+        # seconds
+        second = (
+            pynutil.delete("seconds:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+
+        suffix = (
+            delete_space
+            + insert_space
+            + pynutil.delete("suffix:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        optional_suffix = pynini.closure(suffix, 0, 1)
+        zone = (
+            delete_space
+            + insert_space
+            + pynutil.delete("zone:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        optional_zone = pynini.closure(zone, 0, 1)
+
+        # hms
+        graph_hms = (
+            (hour @ add_leading_zero_to_double_digit)
+            + delete_space
+            + pynutil.insert(":")
+            + (minute @ add_leading_zero_to_double_digit)
+            + delete_space
+            + pynutil.insert(":")
+            + second
+        )
+        # hm
+        graph_hm = (
+            (hour @ add_leading_zero_to_double_digit)
+            + delete_space
+            + pynutil.insert(":")
+            + (minute @ add_leading_zero_to_double_digit)
+        )
+
+        graph = (graph_hms | graph_hm) + optional_suffix + optional_zone
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py

+from fun_text_processing.inverse_text_normalization.ko.verbalizers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.date import DateFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.fraction import FractionFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.ordinal import OrdinalFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.time import TimeFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.whitelist import WhiteListFst
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize", kind="verbalize")
+        cardinal = CardinalFst()
+        cardinal_graph = cardinal.fst
+        ordinal_graph = OrdinalFst().fst
+        decimal = DecimalFst()
+        decimal_graph = decimal.fst
+        fraction = FractionFst()
+        fraction_graph = fraction.fst
+        measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst
+        money_graph = MoneyFst(decimal=decimal).fst
+        time_graph = TimeFst().fst
+        date_graph = DateFst().fst
+        whitelist_graph = WhiteListFst().fst
+        telephone_graph = TelephoneFst().fst
+        electronic_graph = ElectronicFst().fst
+        graph = (
+            time_graph
+            | date_graph
+            | money_graph
+            | measure_graph
+            | ordinal_graph
+            | decimal_graph
+            | fraction_graph
+            | cardinal_graph
+            | whitelist_graph
+            | telephone_graph
+            | electronic_graph
+        )
+        self.fst = graph

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence, e.g.
+    tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize_final", kind="verbalize")
+        verbalize = VerbalizeFst().fst
+        word = WordFst().fst
+        types = verbalize | word
+        graph = (
+            pynutil.delete("tokens")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + types
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+        self.fst = graph

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for verbalizing whitelist
+        e.g. tokens { name: "mrs." } -> mrs.
+    """
+
+    def __init__(self):
+        super().__init__(name="whitelist", kind="verbalize")
+        graph = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+        self.fst = graph.optimize()

FunASR/fun_text_processing/inverse_text_normalization/ko/verbalizers/word.py

+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for verbalizing plain tokens
+        e.g. tokens { name: "sleep" } -> sleep
+    """
+
+    def __init__(self):
+        super().__init__(name="word", kind="verbalize")
+        chars = pynini.closure(DAMO_CHAR - " ", 1)
+        char = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + chars
+            + pynutil.delete('"')
+        )
+        graph = char @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+
+        self.fst = graph.optimize()

FunASR/fun_text_processing/inverse_text_normalization/pt/__init__.py

+from fun_text_processing.inverse_text_normalization.pt.taggers.tokenize_and_classify import (
+    ClassifyFst,
+)
+from fun_text_processing.inverse_text_normalization.pt.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.pt.verbalizers.verbalize_final import (
+    VerbalizeFinalFst,
+)