{
  "slug": "persian-tokenizer",
  "name": "gizza-ai/persian-tokenizer",
  "version": "0.1.0",
  "title": "Persian Tokenizer — Split Farsi Text into Words and Sentences — gizza.ai",
  "description": "Tokenize Persian text into words and sentences. ZWNJ half-space compounds stay whole, Persian punctuation and digits handled. Free, in your browser.",
  "tags": [
    "persian tokenizer",
    "farsi tokenizer",
    "persian word tokenizer",
    "persian sentence tokenizer",
    "farsi text segmentation",
    "zwnj half space",
    "persian nlp",
    "tokenize farsi text",
    "persian word count"
  ],
  "category": "text",
  "urls": {
    "page": "https://gizza.ai/tools/persian-tokenizer/",
    "markdown": "https://gizza.ai/tools/persian-tokenizer/index.md",
    "descriptor": "https://gizza.ai/tools/persian-tokenizer/tool.json",
    "deep_link_example": "https://gizza.ai/tools/persian-tokenizer/?text=%D9%85%D8%A7%20%DA%A9%D8%AA%D8%A7%D8%A8%20%D9%85%DB%8C%E2%80%8C%D8%AE%D9%88%D8%A7%D9%86%DB%8C%D9%85.%20%DB%8C%D8%A7%D8%AF%DA%AF%DB%8C%D8%B1%DB%8C%20%D8%AE%D9%88%D8%A8%20%D8%A7%D8%B3%D8%AA.&mode=words&format=lines&punctuation=separate&split_zwnj=true&normalize=true&keep_entities=true&newlines=paragraph"
  },
  "cli": "gizza tool persian-tokenizer \"ما کتاب می‌خوانیم. یادگیری خوب است.\"",
  "tool": {
    "description": "Tokenize Persian/Farsi text into words and sentences with a deterministic rule-based segmenter. ZWNJ half-space compounds (می‌خوانیم, کتاب‌ها) stay one word by default and split into their parts with split_zwnj=true. Persian punctuation (، ؛ ؟ « » ۔) is recognised alongside the ASCII marks, and ؟ ۔ ⸮ end a sentence just like ? and .; a period between digits or inside a URL/email never does. All three digit sets (0-9, ٠-٩, ۰-۹) count as digits so ۱۳۹۶/۰۶/۱۱ and 1,250.75 stay whole, as do URLs, emails, @mentions and #hashtags (keep_entities, default true). normalize (default true) folds Arabic ي/ك/ى/ة to Persian and strips harakat and the kashida first. Choose mode='words' (default), 'sentences' or 'both'; format='lines' (default), 'numbered', 'space-separated' or 'json' with sentence and token counts; punctuation='separate' (default), 'attach' or 'remove'; newlines='paragraph' (default), 'never' or 'always'.",
    "parameters": {
      "additionalProperties": false,
      "properties": {
        "format": {
          "default": "lines",
          "description": "How to render the result. 'lines' (default) = one item per line; 'numbered' = one per line prefixed '1. ', '2. '; 'space-separated' = items joined by a single space (the classic tokenizer output 'ما کتاب می‌خوانیم'); 'json' = {\"mode\",\"sentence_count\",\"token_count\",\"tokens\"|\"sentences\"}.",
          "enum": [
            "lines",
            "numbered",
            "space-separated",
            "json"
          ],
          "type": "string"
        },
        "keep_entities": {
          "default": true,
          "description": "Keep URLs, email addresses, @mentions, #hashtags and separator-bearing numbers whole: 'https://example.com/a', 'info@example.com', '۱۳۹۶/۰۶/۱۱', '1,250.75' each stay one token. Default true; set false to split them at every separator.",
          "type": "boolean"
        },
        "mode": {
          "default": "words",
          "description": "What to return. 'words' (default) = word tokens for the whole text; 'sentences' = sentence segments only; 'both' = each sentence followed by its own word tokens.",
          "enum": [
            "words",
            "sentences",
            "both"
          ],
          "type": "string"
        },
        "newlines": {
          "default": "paragraph",
          "description": "How line breaks affect sentence boundaries. 'paragraph' (default) = only a blank line ends a sentence; 'never' = line breaks are ordinary whitespace, so wrapped lines rejoin; 'always' = every line break ends a sentence (lists, subtitles, one item per line).",
          "enum": [
            "paragraph",
            "never",
            "always"
          ],
          "type": "string"
        },
        "normalize": {
          "default": true,
          "description": "Fold Arabic letter forms to Persian (ي→ی, ك→ک, ى→ی, ة→ه), convert Arabic-Indic digits ٠-٩ to Persian ۰-۹, and strip harakat (تشکیل) plus the kashida ـ before tokenizing, so the same word typed on an Arabic keyboard yields the same token. Default true; set false to keep the original characters.",
          "type": "boolean"
        },
        "punctuation": {
          "default": "separate",
          "description": "What happens to punctuation marks such as ، ؛ ؟ « » . ! ?. 'separate' (default) = each mark is its own token, repeats of one mark grouped ('؟؟', '...'); 'attach' = marks stay glued to the word they touch (split on whitespace only); 'remove' = punctuation tokens are dropped.",
          "enum": [
            "separate",
            "attach",
            "remove"
          ],
          "type": "string"
        },
        "split_zwnj": {
          "default": false,
          "description": "Break ZWNJ half-space compounds (نیم‌فاصله, U+200C) into their parts: 'می‌خوانیم' becomes 'می' + 'خوانیم' and 'کتاب‌ها' becomes 'کتاب' + 'ها'. Default false, which keeps each compound as ONE word — the usual choice for word counts and search indexing.",
          "type": "boolean"
        },
        "text": {
          "description": "The Persian/Farsi text to tokenize. Plain text, up to 200000 characters — e.g. 'ما کتاب می‌خوانیم. یادگیری خوب است.'",
          "type": "string"
        }
      },
      "required": [
        "text"
      ],
      "type": "object"
    }
  }
}