{ "slug": "url-normalizer", "name": "gizza-ai/url-normalizer", "version": "0.1.0", "title": "URL Normalizer — Canonicalize Hosts, Paths and Query Strings — gizza.ai", "description": "Normalize URLs in bulk: lowercase hosts, remove default ports, resolve dot-segments, canonicalize percent-encoding, sort query params, and optionally strip tracking.", "tags": [ "url normalizer", "url canonicalizer", "canonical url", "normalize urls", "sort query parameters", "remove default port", "resolve dot segments", "strip utm", "seo canonical url" ], "category": "network", "urls": { "page": "https://gizza.ai/tools/url-normalizer/", "markdown": "https://gizza.ai/tools/url-normalizer/index.md", "descriptor": "https://gizza.ai/tools/url-normalizer/tool.json", "deep_link_example": "https://gizza.ai/tools/url-normalizer/?urls=HTTP%3A%2F%2FExample.COM%3A80%2Fa%2Fb%2F..%2Fc%3Fb%3D2%26a%3D1%0Ahttps%3A%2F%2Fwww.example.com%2Fblog%2Findex.html%3Futm_source%3Dnews%26id%3D42%23intro%0A..%2Fimages%2Flogo.png&base=https%3A%2F%2Fexample.com%2Fdocs%2Fguide%2Findex.html&scheme=preserve&www=preserve&strip_default_port=true&dot_segments=true&collapse_slashes=true&lowercase_path=true&encoding=normalize&drop_index=true&trailing_slash=preserve&sort_query=key&dedupe_query=true&drop_empty_params=true&drop_tracking=true&drop_fragment=true&dedupe_urls=true&on_invalid=keep&output=urls" }, "cli": "gizza tool url-normalizer \"HTTP://Example.COM:80/a/b/../c?b=2&a=1\nhttps://www.example.com/blog/index.html?utm_source=news&id=42#intro\n../images/logo.png\"", "tool": { "description": "Normalize a batch of URLs to one canonical form, per RFC 3986 syntax-based normalization. Lowercases the scheme and host, drops a port the scheme already implies, resolves '.' and '..' path segments, rewrites percent-encoding to a single canonical spelling, and sorts the query parameters so '?b=2&a=1' and '?a=1&b=2' collapse to the same string. Optional SEO passes on top: force http or https, add or strip 'www.', add or remove trailing slashes, collapse duplicate slashes, lowercase the path, remove directory-index files like index.html, drop tracking parameters, empty parameters, repeated parameters and the fragment, and deduplicate the whole list. Takes one URL per line — absolute, scheme-relative, bare host, or relative against an optional base URL — and returns the normalized list, only the URLs that changed, a per-line CSV report, or a CSV summary of the totals.", "parameters": { "additionalProperties": false, "properties": { "base": { "default": "", "description": "Optional absolute base URL used to turn relative references into absolute ones, per RFC 3986 section 5.2 — e.g. 'https://example.com/docs/guide/index.html' makes '../images/logo.png' resolve to 'https://example.com/docs/images/logo.png'. Lines that already carry their own scheme are left absolute and unaffected. Empty by default, which leaves relative lines relative.", "type": "string" }, "collapse_slashes": { "default": false, "description": "Collapse runs of '/' in the path to a single slash, so '/a//b///c' becomes '/a/b/c'. Off by default: most servers serve both spellings identically, but a few treat an empty segment as meaningful, and RFC 3986 does not license the change. Turn it on when a crawl export is full of accidental double slashes.", "type": "boolean" }, "dedupe_query": { "default": false, "description": "Drop repeated query parameters that are byte-identical after encoding normalization, keeping the first — so '?a=1&a=1' becomes '?a=1' while a genuinely multi-valued 'tag=a&tag=b' survives intact. Off by default because a repeat can be meaningful to some applications.", "type": "boolean" }, "dedupe_urls": { "default": false, "description": "Drop lines that normalize to a URL an earlier line already produced, keeping the first occurrence and the original order. This is the point of canonicalizing a crawl export: 'https://Example.com/a?b=1&a=2' and 'https://example.com/a?a=2&b=1' are one page. Default false.", "type": "boolean" }, "dot_segments": { "default": true, "description": "Resolve '.' and '..' inside the path using the RFC 3986 section 5.2.4 algorithm, so '/a/b/../c' becomes '/a/c' and '/a/./b' becomes '/a/b'. On by default — this is what makes two spellings of the same file compare equal. Turn it off to leave the path segments exactly as written.", "type": "boolean" }, "drop_empty_params": { "default": false, "description": "Remove query parameters with no value — both 'a=' and a bare valueless 'flag'. Off by default, because an empty value is meaningful to some applications; turn it on to clear the leftovers an unfilled form appends to a URL. If every parameter goes, the '?' goes with them.", "type": "boolean" }, "drop_fragment": { "default": false, "description": "Remove the '#fragment' entirely. Off by default, since a fragment is part of the address a user shared. An EMPTY fragment (a bare trailing '#') is always dropped, because it addresses the same resource. Turn this on for the SEO canonical form, where the fragment is never part of the indexed URL.", "type": "boolean" }, "drop_index": { "default": false, "description": "Remove a trailing directory-index file from the path, leaving the directory slash — '/blog/index.html' becomes '/blog/'. Recognizes index.html, index.htm, index.php, index.asp, index.aspx, index.jsp, index.cgi, index.shtml, index.xhtml and the default.* equivalents, matched case-insensitively. Off by default because not every server serves the two forms identically.", "type": "boolean" }, "drop_tracking": { "default": false, "description": "Remove the usual analytics and click-ID query parameters — the utm_*, pk_*, mtm_*, ga_*, _hs* families plus fbclid, gclid, msclkid, yclid, igshid, mkt_tok and friends. Off by default because canonicalizing and stripping are separate decisions. For allow/deny lists and custom parameter names use the dedicated URL query normalizer instead.", "type": "boolean" }, "encoding": { "default": "normalize", "description": "Percent-encoding policy, applied to the path, query and fragment separately. 'normalize' (default) is RFC 3986 section 6.2.2: escapes of unreserved characters (A-Z a-z 0-9 - . _ ~) are decoded to literals, everything that must stay escaped keeps uppercase hex, and illegal literals such as a space are escaped — so '%2d' becomes '-' and '%c3%a9' becomes '%C3%A9'. 'decode' additionally unescapes characters that are legal literals in that component, including '%2F' in a path, which — combined with dot_segments — reveals a disguised traversal like '%2E%2E%2F'. 'preserve' leaves every escape byte-for-byte and only reorders and filters. A lone '%' is never an error: it is escaped as '%25'.", "enum": [ "normalize", "decode", "preserve" ], "type": "string" }, "lowercase_path": { "default": false, "description": "Lowercase the letters in the path, leaving percent-escape hex digits uppercase — '/Docs/Caf%c3%a9' becomes '/docs/caf%C3%A9'. Off by default because a path is case-SENSITIVE on most servers, so this can point a URL at a page that does not exist. Turn it on only for a case-insensitive host such as IIS. The scheme and host are always lowercased regardless.", "type": "boolean" }, "on_invalid": { "default": "keep", "description": "What to do with a line that cannot be parsed — a host containing a space or an illegal character, an unterminated IPv6 literal, a non-numeric port, a control character. 'keep' (default) passes it through untouched so an annotated list survives a round trip, 'drop' leaves it out of the result, 'error' fails the run and names the line number and the reason.", "enum": [ "keep", "drop", "error" ], "type": "string" }, "output": { "default": "urls", "description": "What to return: 'urls' (default) is every line normalized, one per line; 'changed' is only the URLs that actually differ from the input, which is the canonical/redirect list worth acting on; 'report' is a line,original,normalized,action CSV covering every input line, where action is normalized, unchanged, duplicate or invalid; 'summary' is a metric,value CSV of the run totals.", "enum": [ "urls", "changed", "report", "summary" ], "type": "string" }, "scheme": { "default": "preserve", "description": "What to do with the scheme. It is always lowercased ('HTTP:' becomes 'http:'). 'preserve' (default) keeps whichever scheme was written; 'https' and 'http' rewrite it, but ONLY for http/https URLs and for scheme-less lines that name a host — a 'mailto:', 'ftp:' or 'tel:' line is never rewritten into a web URL.", "enum": [ "preserve", "https", "http" ], "type": "string" }, "sort_query": { "default": "key", "description": "How to order the query parameters — this is what makes '?b=2&a=1' and '?a=1&b=2' collapse to one canonical string. 'key' (default) sorts alphabetically by parameter name; 'key-value' also orders repeats of the same name by value; 'none' keeps the original order. Sorting is stable, so equally-ranked parameters keep the order you gave them.", "enum": [ "key", "key-value", "none" ], "type": "string" }, "strip_default_port": { "default": true, "description": "Remove a port that the scheme already implies — ':80' on http, ':443' on https, plus the defaults for ws, wss, ftp, ftps, sftp, ssh, telnet, smtp, gopher, pop3, nntp, imap, ldap and ldaps. On by default (RFC 3986 section 6.2.3). Leading zeros are trimmed first, so ':0443' counts as 443, and an empty ':' is always dropped. Turn it off to keep the port exactly as written.", "type": "boolean" }, "trailing_slash": { "default": "preserve", "description": "Trailing-slash policy for the path. 'preserve' (default) leaves it alone; 'add' appends a slash to directory-style paths but never to a last segment that looks like a file ('/sitemap.xml' stays put); 'remove' strips trailing slashes. The site root always stays a single '/'. Applied after drop_index, so '/blog/index.html' with both set becomes '/blog/' or '/blog'.", "enum": [ "preserve", "add", "remove" ], "type": "string" }, "urls": { "description": "The URLs to normalize, one per line — e.g. 'HTTP://Example.COM:80/a/b/../c?b=2&a=1'. Absolute URLs (any scheme), scheme-relative URLs ('//cdn.example.com/a'), bare hosts ('Example.COM/path', 'localhost:8080/a') and — when a base URL is given — relative references ('../images/logo.png', '/about', '?page=2') are all accepted. Blank lines are ignored. Max 20,000 URLs and 1,000,000 bytes per run.", "type": "string" }, "www": { "default": "preserve", "description": "The 'www.' policy for the host. 'preserve' (default) leaves it as written; 'strip' removes a leading 'www.' as long as at least two labels remain, so 'www.example.com' becomes 'example.com' but 'www.com' is left alone; 'add' prefixes 'www.' to a bare apex domain only ('example.com' becomes 'www.example.com'; 'blog.example.com' and IP addresses are untouched). Pick one and use it consistently — the two forms are different hosts to a search engine.", "enum": [ "preserve", "strip", "add" ], "type": "string" } }, "required": [ "urls" ], "type": "object" } } }