Linguifex

Module:headword/page: Difference between revisions

← Older editNewer edit →
No edit summary
No edit summary
Line 1: Line 1:
local require = require
local export = {}
local m_str_utils = require("Module:string utilities")
 
local collation_module = "Module:collation"
local languages_module = "Module:languages"
local maintenance_category_module = "Module:maintenance category"
local string_utilities_module = "Module:string utilities"
local table_module = "Module:table"
local template_parser_module = "Module:template parser"


local mw = mw
local mw = mw
Line 8: Line 14:


local concat = table.concat
local concat = table.concat
local decode_entities = m_str_utils.decode_entities
local encode_entities = m_str_utils.encode_entities
local find = string.find
local find = string.find
local get_category = require("Module:maintenance category").get_category
local format = string.format
local get_etym_lang = require("Module:etymology languages").getByCanonicalName
local gsub = string.gsub
local insert = table.insert
local insert = table.insert
local ipairs = ipairs
local list_to_set = require("Module:table").listToSet
local load_data = mw.loadData
local load_data = mw.loadData
local match = string.match
local match = string.match
local new_title = mw.title.new
local new_title = mw.title.new
local remove_comments = m_str_utils.remove_comments
local pairs = pairs
local split = m_str_utils.split
local require = require
local string_sort = require("Module:collation").string_sort
local sub = string.sub
local sub = string.sub
local toNFC = ustring.toNFC
local toNFC = ustring.toNFC
local toNFD = ustring.toNFD
local toNFD = ustring.toNFD
local type = type
local type_or_class = require("Module:parser").type_or_class
local u = m_str_utils.char
local ugsub = ustring.gsub
local ugsub = ustring.gsub
local uupper = m_str_utils.upper


local langnames = load_data("Module:languages/canonical names")
--[==[
local etym_langnames = load_data("Module:etymology languages/canonical names")
Loaders for functions in other modules, which overwrite themselves with the target function when called. This ensures modules are only loaded when needed, retains the speed/convenience of locally-declared pre-loaded functions, and has no overhead after the first call, since the target functions are called directly in any subsequent calls.]==]
local function class_else_type(...)
class_else_type = require(template_parser_module).class_else_type
return class_else_type(...)
end
local function decode_entities(...)
decode_entities = require(string_utilities_module).decode_entities
return decode_entities(...)
end
local function encode_entities(...)
encode_entities = require(string_utilities_module).encode_entities
return encode_entities(...)
end
local function get_category(...)
get_category = require(maintenance_category_module).get_category
return get_category(...)
end
local function get_lang(...)
get_lang = require(languages_module).getByCode
return get_lang(...)
end
local function list_to_set(...)
list_to_set = require(table_module).listToSet
return list_to_set(...)
end
local function parse(...)
parse = require(template_parser_module).parse
return parse(...)
end
local function remove_comments(...)
remove_comments = require(string_utilities_module).remove_comments
return remove_comments(...)
end
local function split(...)
split = require(string_utilities_module).split
return split(...)
end
local function string_sort(...)
string_sort = require(collation_module).string_sort
return string_sort(...)
end
local function uupper(...)
uupper = require(string_utilities_module).upper
return uupper(...)
end


local export = {}
--[==[
 
Loaders for objects, which load data (or some other object) into some variable, which can then be accessed as "foo or get_foo()", where the function get_foo sets the object to "foo" and then returns it. This ensures they are only loaded when needed, and avoids the need to check for the existence of the object each time, since once "foo" has been set, "get_foo" will not be called again.]==]
-- Convert a numeric list of characters and ranges to the equivalent Lua pattern. WARNING: This destructively modifies
local langnames
-- the contents of `ranges`.
local function get_langnames()
local function char_ranges_to_pattern(ranges)
langnames, get_langnames = load_data("Module:languages/canonical names"), nil
for j, range in ipairs(ranges) do
return langnames
if type(range) == "table" then
for k, char in ipairs(range) do
range[k] = u(char)
end
ranges[j] = concat(range, "-")
else
ranges[j] = u(range)
end
end
end
return concat(ranges)
end


-- Combining character data used when categorising unusual characters. These resolve into two patterns, used to find
-- Combining character data used when categorising unusual characters. These resolve into two patterns, used to find
-- single combining characters (i.e. character + diacritic(s)) or double combining characters (i.e. character +
-- single combining characters (i.e. character + diacritic(s)) or double combining characters (i.e. character +
-- diacritic(s) + character).
-- diacritic(s) + character).
local comb_chars = {
-- Charsets are in the format used by Unicode's UnicodeSet tool: https://util.unicode.org/UnicodeJsps/list-unicodeset.jsp.
single = {
 
{0x0300, 0x034E},
-- Single combining characters.
-- Exclude combining grapheme joiner.
-- Charset: [[:M:]&[:^Canonical_Combining_Class=/^Double_/:]&[:^subhead=Grapheme joiner:]&[:^Variation_Selector=Yes:]]
{0x0350, 0x035B},
-- Note: concatenating hundreds of lines at once gives an error, so () are used every 150 lines to break it up into chunks.
{0x0363, 0x036F},
local comb_chars_single =
{0x0483, 0x0489},
("\204\128-\205\142" .. -- U+0300-U+034E
{0x0591, 0x05BD},
"\205\144-\205\155" .. -- U+0350-U+035B
0x05BF,
"\205\163-\205\175" .. -- U+0363-U+036F
{0x05C1, 0x05C2},
"\210\131-\210\137" .. -- U+0483-U+0489
{0x05C4, 0x05C5},
"\214\145-\214\189" .. -- U+0591-U+05BD
0x05C7,
"\214\191" .. -- U+05BF
{0x0610, 0x061A},
"\215\129" .. -- U+05C1
{0x064B, 0x065F},
"\215\130" .. -- U+05C2
0x0670,
"\215\132" .. -- U+05C4
{0x06D6, 0x06DC},
"\215\133" .. -- U+05C5
{0x06DF, 0x06E4},
"\215\135" .. -- U+05C7
{0x06E7, 0x06E8},
"\216\144-\216\154" .. -- U+0610-U+061A
{0x06EA, 0x06ED},
"\217\139-\217\159" .. -- U+064B-U+065F
0x0711,
"\217\176" .. -- U+0670
{0x0730, 0x074A},
"\219\150-\219\156" .. -- U+06D6-U+06DC
{0x07A6, 0x07B0},
"\219\159-\219\164" .. -- U+06DF-U+06E4
{0x07EB, 0x07F3},
"\219\167" .. -- U+06E7
0x07FD,
"\219\168" .. -- U+06E8
{0x0816, 0x0819},
"\219\170-\219\173" .. -- U+06EA-U+06ED
{0x081B, 0x0823},
"\220\145" .. -- U+0711
{0x0825, 0x0827},
"\220\176-\221\138" .. -- U+0730-U+074A
{0x0829, 0x082D},
"\222\166-\222\176" .. -- U+07A6-U+07B0
{0x0859, 0x085B},
"\223\171-\223\179" .. -- U+07EB-U+07F3
{0x0898, 0x089F},
"\223\189" .. -- U+07FD
{0x08CA, 0x08E1},
"\224\160\150-\224\160\153" .. -- U+0816-U+0819
{0x08E3, 0x0903},
"\224\160\155-\224\160\163" .. -- U+081B-U+0823
{0x093A, 0x093C},
"\224\160\165-\224\160\167" .. -- U+0825-U+0827
{0x093E, 0x094F},
"\224\160\169-\224\160\173" .. -- U+0829-U+082D
{0x0951, 0x0957},
"\224\161\153-\224\161\155" .. -- U+0859-U+085B
{0x0962, 0x0963},
"\224\162\151-\224\162\159" .. -- U+0897-U+089F
{0x0981, 0x0983},
"\224\163\138-\224\163\161" .. -- U+08CA-U+08E1
0x09BC,
"\224\163\163-\224\164\131" .. -- U+08E3-U+0903
{0x09BE, 0x09C4},
"\224\164\186-\224\164\188" .. -- U+093A-U+093C
{0x09C7, 0x09C8},
"\224\164\190-\224\165\143" .. -- U+093E-U+094F
{0x09CB, 0x09CD},
"\224\165\145-\224\165\151" .. -- U+0951-U+0957
0x09D7,
"\224\165\162" .. -- U+0962
{0x09E2, 0x09E3},
"\224\165\163" .. -- U+0963
0x09FE,
"\224\166\129-\224\166\131" .. -- U+0981-U+0983
{0x0A01, 0x0A03},
"\224\166\188" .. -- U+09BC
0x0A3C,
"\224\166\190-\224\167\132" .. -- U+09BE-U+09C4
{0x0A3E, 0x0A42},
"\224\167\135" .. -- U+09C7
{0x0A47, 0x0A48},
"\224\167\136" .. -- U+09C8
{0x0A4B, 0x0A4D},
"\224\167\139-\224\167\141" .. -- U+09CB-U+09CD
0x0A51,
"\224\167\151" .. -- U+09D7
{0x0A70, 0x0A71},
"\224\167\162" .. -- U+09E2
0x0A75,
"\224\167\163" .. -- U+09E3
{0x0A81, 0x0A83},
"\224\167\190" .. -- U+09FE
0x0ABC,
"\224\168\129-\224\168\131" .. -- U+0A01-U+0A03
{0x0ABE, 0x0AC5},
"\224\168\188" .. -- U+0A3C
{0x0AC7, 0x0AC9},
"\224\168\190-\224\169\130" .. -- U+0A3E-U+0A42
{0x0ACB, 0x0ACD},
"\224\169\135" .. -- U+0A47
{0x0AE2, 0x0AE3},
"\224\169\136" .. -- U+0A48
{0x0AFA, 0x0AFF},
"\224\169\139-\224\169\141" .. -- U+0A4B-U+0A4D
{0x0B01, 0x0B03},
"\224\169\145" .. -- U+0A51
0x0B3C,
"\224\169\176" .. -- U+0A70
{0x0B3E, 0x0B44},
"\224\169\177" .. -- U+0A71
{0x0B47, 0x0B48},
"\224\169\181" .. -- U+0A75
{0x0B4B, 0x0B4D},
"\224\170\129-\224\170\131" .. -- U+0A81-U+0A83
{0x0B55, 0x0B57},
"\224\170\188" .. -- U+0ABC
{0x0B62, 0x0B63},
"\224\170\190-\224\171\133" .. -- U+0ABE-U+0AC5
0x0B82,
"\224\171\135-\224\171\137" .. -- U+0AC7-U+0AC9
{0x0BBE, 0x0BC2},
"\224\171\139-\224\171\141" .. -- U+0ACB-U+0ACD
{0x0BC6, 0x0BC8},
"\224\171\162" .. -- U+0AE2
{0x0BCA, 0x0BCD},
"\224\171\163" .. -- U+0AE3
0x0BD7,
"\224\171\186-\224\171\191" .. -- U+0AFA-U+0AFF
{0x0C00, 0x0C04},
"\224\172\129-\224\172\131" .. -- U+0B01-U+0B03
0x0C3C,
"\224\172\188" .. -- U+0B3C
{0x0C3E, 0x0C44},
"\224\172\190-\224\173\132" .. -- U+0B3E-U+0B44
{0x0C46, 0x0C48},
"\224\173\135" .. -- U+0B47
{0x0C4A, 0x0C4D},
"\224\173\136" .. -- U+0B48
{0x0C55, 0x0C56},
"\224\173\139-\224\173\141" .. -- U+0B4B-U+0B4D
{0x0C62, 0x0C63},
"\224\173\149-\224\173\151" .. -- U+0B55-U+0B57
{0x0C81, 0x0C83},
"\224\173\162" .. -- U+0B62
0x0CBC,
"\224\173\163" .. -- U+0B63
{0x0CBE, 0x0CC4},
"\224\174\130" .. -- U+0B82
{0x0CC6, 0x0CC8},
"\224\174\190-\224\175\130" .. -- U+0BBE-U+0BC2
{0x0CCA, 0x0CCD},
"\224\175\134-\224\175\136" .. -- U+0BC6-U+0BC8
{0x0CD5, 0x0CD6},
"\224\175\138-\224\175\141" .. -- U+0BCA-U+0BCD
{0x0CE2, 0x0CE3},
"\224\175\151" .. -- U+0BD7
0x0CF3,
"\224\176\128-\224\176\132" .. -- U+0C00-U+0C04
{0x0D00, 0x0D03},
"\224\176\188" .. -- U+0C3C
{0x0D3B, 0x0D3C},
"\224\176\190-\224\177\132" .. -- U+0C3E-U+0C44
{0x0D3E, 0x0D44},
"\224\177\134-\224\177\136" .. -- U+0C46-U+0C48
{0x0D46, 0x0D48},
"\224\177\138-\224\177\141" .. -- U+0C4A-U+0C4D
{0x0D4A, 0x0D4D},
"\224\177\149" .. -- U+0C55
0x0D57,
"\224\177\150" .. -- U+0C56
{0x0D62, 0x0D63},
"\224\177\162" .. -- U+0C62
{0x0D81, 0x0D83},
"\224\177\163" .. -- U+0C63
0x0DCA,
"\224\178\129-\224\178\131" .. -- U+0C81-U+0C83
{0x0DCF, 0x0DD4},
"\224\178\188" .. -- U+0CBC
0x0DD6,
"\224\178\190-\224\179\132" .. -- U+0CBE-U+0CC4
{0x0DD8, 0x0DDF},
"\224\179\134-\224\179\136" .. -- U+0CC6-U+0CC8
{0x0DF2, 0x0DF3},
"\224\179\138-\224\179\141" .. -- U+0CCA-U+0CCD
0x0E31,
"\224\179\149" .. -- U+0CD5
{0x0E34, 0x0E3A},
"\224\179\150" .. -- U+0CD6
{0x0E47, 0x0E4E},
"\224\179\162" .. -- U+0CE2
0x0EB1,
"\224\179\163" .. -- U+0CE3
{0x0EB4, 0x0EBC},
"\224\179\179" .. -- U+0CF3
{0x0EC8, 0x0ECE},
"\224\180\128-\224\180\131" .. -- U+0D00-U+0D03
{0x0F18, 0x0F19},
"\224\180\187" .. -- U+0D3B
0x0F35,
"\224\180\188" .. -- U+0D3C
0x0F37,
"\224\180\190-\224\181\132" .. -- U+0D3E-U+0D44
0x0F39,
"\224\181\134-\224\181\136" .. -- U+0D46-U+0D48
{0x0F3E, 0x0F3F},
"\224\181\138-\224\181\141" .. -- U+0D4A-U+0D4D
{0x0F71, 0x0F84},
"\224\181\151" .. -- U+0D57
{0x0F86, 0x0F87},
"\224\181\162" .. -- U+0D62
{0x0F8D, 0x0F97},
"\224\181\163" .. -- U+0D63
{0x0F99, 0x0FBC},
"\224\182\129-\224\182\131" .. -- U+0D81-U+0D83
0x0FC6,
"\224\183\138" .. -- U+0DCA
{0x102B, 0x103E},
"\224\183\143-\224\183\148" .. -- U+0DCF-U+0DD4
{0x1056, 0x1059},
"\224\183\150" .. -- U+0DD6
{0x105E, 0x1060},
"\224\183\152-\224\183\159" .. -- U+0DD8-U+0DDF
{0x1062, 0x1064},
"\224\183\178" .. -- U+0DF2
{0x1067, 0x106D},
"\224\183\179" .. -- U+0DF3
{0x1071, 0x1074},
"\224\184\177" .. -- U+0E31
{0x1082, 0x108D},
"\224\184\180-\224\184\186" .. -- U+0E34-U+0E3A
0x108F,
"\224\185\135-\224\185\142" .. -- U+0E47-U+0E4E
{0x109A, 0x109D},
"\224\186\177" .. -- U+0EB1
{0x135D, 0x135F},
"\224\186\180-\224\186\188" .. -- U+0EB4-U+0EBC
{0x1712, 0x1715},
"\224\187\136-\224\187\142" .. -- U+0EC8-U+0ECE
{0x1732, 0x1734},
"\224\188\152" .. -- U+0F18
{0x1752, 0x1753},
"\224\188\153" .. -- U+0F19
{0x1772, 0x1773},
"\224\188\181" .. -- U+0F35
{0x17B4, 0x17D3},
"\224\188\183" .. -- U+0F37
0x17DD,
"\224\188\185" .. -- U+0F39
-- Exclude Mongolian variation selectors.
"\224\188\190" .. -- U+0F3E
{0x1885, 0x1886},
"\224\188\191" .. -- U+0F3F
0x18A9,
"\224\189\177-\224\190\132" .. -- U+0F71-U+0F84
{0x1920, 0x192B},
"\224\190\134" .. -- U+0F86
{0x1930, 0x193B},
"\224\190\135" .. -- U+0F87
{0x1A17, 0x1A1B},
"\224\190\141-\224\190\151" .. -- U+0F8D-U+0F97
{0x1A55, 0x1A5E},
"\224\190\153-\224\190\188" .. -- U+0F99-U+0FBC
{0x1A60, 0x1A7C},
"\224\191\134" .. -- U+0FC6
0x1A7F,
"\225\128\171-\225\128\190" .. -- U+102B-U+103E
{0x1AB0, 0x1ACE},
"\225\129\150-\225\129\153" .. -- U+1056-U+1059
{0x1B00, 0x1B04},
"\225\129\158-\225\129\160" .. -- U+105E-U+1060
{0x1B34, 0x1B44},
"\225\129\162-\225\129\164" .. -- U+1062-U+1064
{0x1B6B, 0x1B73},
"\225\129\167-\225\129\173" .. -- U+1067-U+106D
{0x1B80, 0x1B82},
"\225\129\177-\225\129\180" .. -- U+1071-U+1074
{0x1BA1, 0x1BAD},
"\225\130\130-\225\130\141" .. -- U+1082-U+108D
{0x1BE6, 0x1BF3},
"\225\130\143" .. -- U+108F
{0x1C24, 0x1C37},
"\225\130\154-\225\130\157" .. -- U+109A-U+109D
{0x1CD0, 0x1CD2},
"\225\141\157-\225\141\159" .. -- U+135D-U+135F
{0x1CD4, 0x1CE8},
"\225\156\146-\225\156\149" .. -- U+1712-U+1715
0x1CED,
"\225\156\178-\225\156\180" .. -- U+1732-U+1734
0x1CF4,
"\225\157\146" .. -- U+1752
{0x1CF7, 0x1CF9},
"\225\157\147" .. -- U+1753
{0x1DC0, 0x1DCC},
"\225\157\178" .. -- U+1772
{0x1DCE, 0x1DFB},
"\225\157\179" .. -- U+1773
{0x1DFD, 0x1DFF},
"\225\158\180-\225\159\147") .. -- U+17B4-U+17D3
{0x20D0, 0x20F0},
("\225\159\157" .. -- U+17DD
{0x2CEF, 0x2CF1},
"\225\162\133" .. -- U+1885
0x2D7F,
"\225\162\134" .. -- U+1886
{0x2DE0, 0x2DFF},
"\225\162\169" .. -- U+18A9
{0x302A, 0x302F},
"\225\164\160-\225\164\171" .. -- U+1920-U+192B
{0x3099, 0x309A},
"\225\164\176-\225\164\187" .. -- U+1930-U+193B
{0xA66F, 0xA672},
"\225\168\151-\225\168\155" .. -- U+1A17-U+1A1B
{0xA674, 0xA67D},
"\225\169\149-\225\169\158" .. -- U+1A55-U+1A5E
{0xA69E, 0xA69F},
"\225\169\160-\225\169\188" .. -- U+1A60-U+1A7C
{0xA6F0, 0xA6F1},
"\225\169\191" .. -- U+1A7F
0xA802,
"\225\170\176-\225\171\142" .. -- U+1AB0-U+1ACE
0xA806,
"\225\172\128-\225\172\132" .. -- U+1B00-U+1B04
0xA80B,
"\225\172\180-\225\173\132" .. -- U+1B34-U+1B44
{0xA823, 0xA827},
"\225\173\171-\225\173\179" .. -- U+1B6B-U+1B73
0xA82C,
"\225\174\128-\225\174\130" .. -- U+1B80-U+1B82
{0xA880, 0xA881},
"\225\174\161-\225\174\173" .. -- U+1BA1-U+1BAD
{0xA8B4, 0xA8C5},
"\225\175\166-\225\175\179" .. -- U+1BE6-U+1BF3
{0xA8E0, 0xA8F1},
"\225\176\164-\225\176\183" .. -- U+1C24-U+1C37
0xA8FF,
"\225\179\144-\225\179\146" .. -- U+1CD0-U+1CD2
{0xA926, 0xA92D},
"\225\179\148-\225\179\168" .. -- U+1CD4-U+1CE8
{0xA947, 0xA953},
"\225\179\173" .. -- U+1CED
{0xA980, 0xA983},
"\225\179\180" .. -- U+1CF4
{0xA9B3, 0xA9C0},
"\225\179\183-\225\179\185" .. -- U+1CF7-U+1CF9
0xA9E5,
"\225\183\128-\225\183\140" .. -- U+1DC0-U+1DCC
{0xAA29, 0xAA36},
"\225\183\142-\225\183\187" .. -- U+1DCE-U+1DFB
0xAA43,
"\225\183\189-\225\183\191" .. -- U+1DFD-U+1DFF
{0xAA4C, 0xAA4D},
"\226\131\144-\226\131\176" .. -- U+20D0-U+20F0
{0xAA7B, 0xAA7D},
"\226\179\175-\226\179\177" .. -- U+2CEF-U+2CF1
0xAAB0,
"\226\181\191" .. -- U+2D7F
{0xAAB2, 0xAAB4},
"\226\183\160-\226\183\191" .. -- U+2DE0-U+2DFF
{0xAAB7, 0xAAB8},
"\227\128\170-\227\128\175" .. -- U+302A-U+302F
{0xAABE, 0xAABF},
"\227\130\153" .. -- U+3099
0xAAC1,
"\227\130\154" .. -- U+309A
{0xAAEB, 0xAAEF},
"\234\153\175-\234\153\178" .. -- U+A66F-U+A672
{0xAAF5, 0xAAF6},
"\234\153\180-\234\153\189" .. -- U+A674-U+A67D
{0xABE3, 0xABEA},
"\234\154\158" .. -- U+A69E
{0xABEC, 0xABED},
"\234\154\159" .. -- U+A69F
0xFB1E,
"\234\155\176" .. -- U+A6F0
{0xFE20, 0xFE2F},
"\234\155\177" .. -- U+A6F1
0x101FD,
"\234\160\130" .. -- U+A802
0x102E0,
"\234\160\134" .. -- U+A806
{0x10376, 0x1037A},
"\234\160\139" .. -- U+A80B
{0x10A01, 0x10A03},
"\234\160\163-\234\160\167" .. -- U+A823-U+A827
{0x10A05, 0x10A06},
"\234\160\172" .. -- U+A82C
{0x10A0C, 0x10A0F},
"\234\162\128" .. -- U+A880
{0x10A38, 0x10A3A},
"\234\162\129" .. -- U+A881
0x10A3F,
"\234\162\180-\234\163\133" .. -- U+A8B4-U+A8C5
{0x10AE5, 0x10AE6},
"\234\163\160-\234\163\177" .. -- U+A8E0-U+A8F1
{0x10D24, 0x10D27},
"\234\163\191" .. -- U+A8FF
{0x10EAB, 0x10EAC},
"\234\164\166-\234\164\173" .. -- U+A926-U+A92D
{0x10EFD, 0x10EFF},
"\234\165\135-\234\165\147" .. -- U+A947-U+A953
{0x10F46, 0x10F50},
"\234\166\128-\234\166\131" .. -- U+A980-U+A983
{0x10F82, 0x10F85},
"\234\166\179-\234\167\128" .. -- U+A9B3-U+A9C0
{0x11000, 0x11002},
"\234\167\165" .. -- U+A9E5
{0x11038, 0x11046},
"\234\168\169-\234\168\182" .. -- U+AA29-U+AA36
0x11070,
"\234\169\131" .. -- U+AA43
{0x11073, 0x11074},
"\234\169\140" .. -- U+AA4C
{0x1107F, 0x11082},
"\234\169\141" .. -- U+AA4D
{0x110B0, 0x110BA},
"\234\169\187-\234\169\189" .. -- U+AA7B-U+AA7D
0x110C2,
"\234\170\176" .. -- U+AAB0
{0x11100, 0x11102},
"\234\170\178-\234\170\180" .. -- U+AAB2-U+AAB4
{0x11127, 0x11134},
"\234\170\183" .. -- U+AAB7
{0x11145, 0x11146},
"\234\170\184" .. -- U+AAB8
0x11173,
"\234\170\190" .. -- U+AABE
{0x11180, 0x11182},
"\234\170\191" .. -- U+AABF
{0x111B3, 0x111C0},
"\234\171\129" .. -- U+AAC1
{0x111C9, 0x111CC},
"\234\171\171-\234\171\175" .. -- U+AAEB-U+AAEF
{0x111CE, 0x111CF},
"\234\171\181" .. -- U+AAF5
{0x1122C, 0x11237},
"\234\171\182" .. -- U+AAF6
0x1123E,
"\234\175\163-\234\175\170" .. -- U+ABE3-U+ABEA
0x11241,
"\234\175\172" .. -- U+ABEC
{0x112DF, 0x112EA},
"\234\175\173" .. -- U+ABED
{0x11300, 0x11303},
"\239\172\158" .. -- U+FB1E
{0x1133B, 0x1133C},
"\239\184\160-\239\184\175" .. -- U+FE20-U+FE2F
{0x1133E, 0x11344},
"\240\144\135\189" .. -- U+101FD
{0x11347, 0x11348},
"\240\144\139\160" .. -- U+102E0
{0x1134B, 0x1134D},
"\240\144\141\182-\240\144\141\186" .. -- U+10376-U+1037A
0x11357,
"\240\144\168\129-\240\144\168\131" .. -- U+10A01-U+10A03
{0x11362, 0x11363},
"\240\144\168\133" .. -- U+10A05
{0x11366, 0x1136C},
"\240\144\168\134" .. -- U+10A06
{0x11370, 0x11374},
"\240\144\168\140-\240\144\168\143" .. -- U+10A0C-U+10A0F
{0x11435, 0x11446},
"\240\144\168\184-\240\144\168\186" .. -- U+10A38-U+10A3A
0x1145E,
"\240\144\168\191" .. -- U+10A3F
{0x114B0, 0x114C3},
"\240\144\171\165" .. -- U+10AE5
{0x115AF, 0x115B5},
"\240\144\171\166" .. -- U+10AE6
{0x115B8, 0x115C0},
"\240\144\180\164-\240\144\180\167" .. -- U+10D24-U+10D27
{0x115DC, 0x115DD},
"\240\144\181\169-\240\144\181\173" .. -- U+10D69-U+10D6D
{0x11630, 0x11640},
"\240\144\186\171" .. -- U+10EAB
{0x116AB, 0x116B7},
"\240\144\186\172" .. -- U+10EAC
{0x1171D, 0x1172B},
"\240\144\187\188-\240\144\187\191" .. -- U+10EFC-U+10EFF
{0x1182C, 0x1183A},
"\240\144\189\134-\240\144\189\144" .. -- U+10F46-U+10F50
{0x11930, 0x11935},
"\240\144\190\130-\240\144\190\133" .. -- U+10F82-U+10F85
{0x11937, 0x11938},
"\240\145\128\128-\240\145\128\130" .. -- U+11000-U+11002
{0x1193B, 0x1193E},
"\240\145\128\184-\240\145\129\134" .. -- U+11038-U+11046
0x11940,
"\240\145\129\176" .. -- U+11070
{0x11942, 0x11943},
"\240\145\129\179" .. -- U+11073
{0x119D1, 0x119D7},
"\240\145\129\180" .. -- U+11074
{0x119DA, 0x119E0},
"\240\145\129\191-\240\145\130\130" .. -- U+1107F-U+11082
0x119E4,
"\240\145\130\176-\240\145\130\186" .. -- U+110B0-U+110BA
{0x11A01, 0x11A0A},
"\240\145\131\130" .. -- U+110C2
{0x11A33, 0x11A39},
"\240\145\132\128-\240\145\132\130" .. -- U+11100-U+11102
{0x11A3B, 0x11A3E},
"\240\145\132\167-\240\145\132\180" .. -- U+11127-U+11134
0x11A47,
"\240\145\133\133" .. -- U+11145
{0x11A51, 0x11A5B},
"\240\145\133\134" .. -- U+11146
{0x11A8A, 0x11A99},
"\240\145\133\179" .. -- U+11173
{0x11C2F, 0x11C36},
"\240\145\134\128-\240\145\134\130" .. -- U+11180-U+11182
{0x11C38, 0x11C3F},
"\240\145\134\179-\240\145\135\128" .. -- U+111B3-U+111C0
{0x11C92, 0x11CA7},
"\240\145\135\137-\240\145\135\140" .. -- U+111C9-U+111CC
{0x11CA9, 0x11CB6},
"\240\145\135\142" .. -- U+111CE
{0x11D31, 0x11D36},
"\240\145\135\143" .. -- U+111CF
0x11D3A,
"\240\145\136\172-\240\145\136\183" .. -- U+1122C-U+11237
{0x11D3C, 0x11D3D},
"\240\145\136\190" .. -- U+1123E
{0x11D3F, 0x11D45},
"\240\145\137\129" .. -- U+11241
0x11D47,
"\240\145\139\159-\240\145\139\170" .. -- U+112DF-U+112EA
{0x11D8A, 0x11D8E},
"\240\145\140\128-\240\145\140\131" .. -- U+11300-U+11303
{0x11D90, 0x11D91},
"\240\145\140\187" .. -- U+1133B
{0x11D93, 0x11D97},
"\240\145\140\188" .. -- U+1133C
{0x11EF3, 0x11EF6},
"\240\145\140\190-\240\145\141\132" .. -- U+1133E-U+11344
{0x11F00, 0x11F01},
"\240\145\141\135" .. -- U+11347
0x11F03,
"\240\145\141\136" .. -- U+11348
{0x11F34, 0x11F3A},
"\240\145\141\139-\240\145\141\141" .. -- U+1134B-U+1134D
{0x11F3E, 0x11F42},
"\240\145\141\151" .. -- U+11357
0x13440,
"\240\145\141\162" .. -- U+11362
{0x13447, 0x13455},
"\240\145\141\163" .. -- U+11363
{0x16AF0, 0x16AF4},
"\240\145\141\166-\240\145\141\172" .. -- U+11366-U+1136C
{0x16B30, 0x16B36},
"\240\145\141\176-\240\145\141\180" .. -- U+11370-U+11374
0x16F4F,
"\240\145\142\184-\240\145\143\128" .. -- U+113B8-U+113C0
{0x16F51, 0x16F87},
"\240\145\143\130" .. -- U+113C2
{0x16F8F, 0x16F92},
"\240\145\143\133" .. -- U+113C5
-- Exclude Khitan Small Script filler.
"\240\145\143\135-\240\145\143\138" .. -- U+113C7-U+113CA
{0x16FF0, 0x16FF1},
"\240\145\143\140-\240\145\143\144" .. -- U+113CC-U+113D0
{0x1BC9D, 0x1BC9E},
"\240\145\143\146" .. -- U+113D2
{0x1CF00, 0x1CF2D},
"\240\145\143\161" .. -- U+113E1
{0x1CF30, 0x1CF46},
"\240\145\143\162" .. -- U+113E2
{0x1D165, 0x1D169},
"\240\145\144\181-\240\145\145\134" .. -- U+11435-U+11446
{0x1D16D, 0x1D172},
"\240\145\145\158" .. -- U+1145E
{0x1D17B, 0x1D182},
"\240\145\146\176-\240\145\147\131" .. -- U+114B0-U+114C3
{0x1D185, 0x1D18B},
"\240\145\150\175-\240\145\150\181" .. -- U+115AF-U+115B5
{0x1D1AA, 0x1D1AD},
"\240\145\150\184-\240\145\151\128" .. -- U+115B8-U+115C0
{0x1D242, 0x1D244},
"\240\145\151\156" .. -- U+115DC
{0x1DA00, 0x1DA36},
"\240\145\151\157" .. -- U+115DD
{0x1DA3B, 0x1DA6C},
"\240\145\152\176-\240\145\153\128" .. -- U+11630-U+11640
0x1DA75,
"\240\145\154\171-\240\145\154\183" .. -- U+116AB-U+116B7
0x1DA84,
"\240\145\156\157-\240\145\156\171" .. -- U+1171D-U+1172B
{0x1DA9B, 0x1DA9F},
"\240\145\160\172-\240\145\160\186" .. -- U+1182C-U+1183A
{0x1DAA1, 0x1DAAF},
"\240\145\164\176-\240\145\164\181" .. -- U+11930-U+11935
{0x1E000, 0x1E006},
"\240\145\164\183" .. -- U+11937
{0x1E008, 0x1E018},
"\240\145\164\184" .. -- U+11938
{0x1E01B, 0x1E021},
"\240\145\164\187-\240\145\164\190" .. -- U+1193B-U+1193E
{0x1E023, 0x1E024},
"\240\145\165\128") .. -- U+11940
{0x1E026, 0x1E02A},
("\240\145\165\130" .. -- U+11942
0x1E08F,
"\240\145\165\131" .. -- U+11943
{0x1E130, 0x1E136},
"\240\145\167\145-\240\145\167\151" .. -- U+119D1-U+119D7
0x1E2AE,
"\240\145\167\154-\240\145\167\160" .. -- U+119DA-U+119E0
{0x1E2EC, 0x1E2EF},
"\240\145\167\164" .. -- U+119E4
{0x1E4EC, 0x1E4EF},
"\240\145\168\129-\240\145\168\138" .. -- U+11A01-U+11A0A
{0x1E8D0, 0x1E8D6},
"\240\145\168\179-\240\145\168\185" .. -- U+11A33-U+11A39
{0x1E944, 0x1E94A},
"\240\145\168\187-\240\145\168\190" .. -- U+11A3B-U+11A3E
},
"\240\145\169\135" .. -- U+11A47
double = {
"\240\145\169\145-\240\145\169\155" .. -- U+11A51-U+11A5B
{0x035C, 0x0362},
"\240\145\170\138-\240\145\170\153" .. -- U+11A8A-U+11A99
0x1DCD,
"\240\145\176\175-\240\145\176\182" .. -- U+11C2F-U+11C36
0x1DFC,
"\240\145\176\184-\240\145\176\191" .. -- U+11C38-U+11C3F
},
"\240\145\178\146-\240\145\178\167" .. -- U+11C92-U+11CA7
vs = { -- variation selectors; separated out so that we don't get categories for them
"\240\145\178\169-\240\145\178\182" .. -- U+11CA9-U+11CB6
{0xFE00, 0xFE0F},
"\240\145\180\177-\240\145\180\182" .. -- U+11D31-U+11D36
{0xE0100, 0xE01EF},
"\240\145\180\186" .. -- U+11D3A
}
"\240\145\180\188" .. -- U+11D3C
}
"\240\145\180\189" .. -- U+11D3D
for key, charset in pairs(comb_chars) do
"\240\145\180\191-\240\145\181\133" .. -- U+11D3F-U+11D45
comb_chars[key] = char_ranges_to_pattern(charset)
"\240\145\181\135" .. -- U+11D47
end
"\240\145\182\138-\240\145\182\142" .. -- U+11D8A-U+11D8E
"\240\145\182\144" .. -- U+11D90
"\240\145\182\145" .. -- U+11D91
"\240\145\182\147-\240\145\182\151" .. -- U+11D93-U+11D97
"\240\145\187\179-\240\145\187\182" .. -- U+11EF3-U+11EF6
"\240\145\188\128" .. -- U+11F00
"\240\145\188\129" .. -- U+11F01
"\240\145\188\131" .. -- U+11F03
"\240\145\188\180-\240\145\188\186" .. -- U+11F34-U+11F3A
"\240\145\188\190-\240\145\189\130" .. -- U+11F3E-U+11F42
"\240\145\189\154" .. -- U+11F5A
"\240\147\145\128" .. -- U+13440
"\240\147\145\135-\240\147\145\149" .. -- U+13447-U+13455
"\240\150\132\158-\240\150\132\175" .. -- U+1611E-U+1612F
"\240\150\171\176-\240\150\171\180" .. -- U+16AF0-U+16AF4
"\240\150\172\176-\240\150\172\182" .. -- U+16B30-U+16B36
"\240\150\189\143" .. -- U+16F4F
"\240\150\189\145-\240\150\190\135" .. -- U+16F51-U+16F87
"\240\150\190\143-\240\150\190\146" .. -- U+16F8F-U+16F92
"\240\150\191\164" .. -- U+16FE4
"\240\150\191\176" .. -- U+16FF0
"\240\150\191\177" .. -- U+16FF1
"\240\155\178\157" .. -- U+1BC9D
"\240\155\178\158" .. -- U+1BC9E
"\240\156\188\128-\240\156\188\173" .. -- U+1CF00-U+1CF2D
"\240\156\188\176-\240\156\189\134" .. -- U+1CF30-U+1CF46
"\240\157\133\165-\240\157\133\169" .. -- U+1D165-U+1D169
"\240\157\133\173-\240\157\133\178" .. -- U+1D16D-U+1D172
"\240\157\133\187-\240\157\134\130" .. -- U+1D17B-U+1D182
"\240\157\134\133-\240\157\134\139" .. -- U+1D185-U+1D18B
"\240\157\134\170-\240\157\134\173" .. -- U+1D1AA-U+1D1AD
"\240\157\137\130-\240\157\137\132" .. -- U+1D242-U+1D244
"\240\157\168\128-\240\157\168\182" .. -- U+1DA00-U+1DA36
"\240\157\168\187-\240\157\169\172" .. -- U+1DA3B-U+1DA6C
"\240\157\169\181" .. -- U+1DA75
"\240\157\170\132" .. -- U+1DA84
"\240\157\170\155-\240\157\170\159" .. -- U+1DA9B-U+1DA9F
"\240\157\170\161-\240\157\170\175" .. -- U+1DAA1-U+1DAAF
"\240\158\128\128-\240\158\128\134" .. -- U+1E000-U+1E006
"\240\158\128\136-\240\158\128\152" .. -- U+1E008-U+1E018
"\240\158\128\155-\240\158\128\161" .. -- U+1E01B-U+1E021
"\240\158\128\163" .. -- U+1E023
"\240\158\128\164" .. -- U+1E024
"\240\158\128\166-\240\158\128\170" .. -- U+1E026-U+1E02A
"\240\158\130\143" .. -- U+1E08F
"\240\158\132\176-\240\158\132\182" .. -- U+1E130-U+1E136
"\240\158\138\174" .. -- U+1E2AE
"\240\158\139\172-\240\158\139\175" .. -- U+1E2EC-U+1E2EF
"\240\158\147\172-\240\158\147\175" .. -- U+1E4EC-U+1E4EF
"\240\158\151\174" .. -- U+1E5EE
"\240\158\151\175" .. -- U+1E5EF
"\240\158\163\144-\240\158\163\150" .. -- U+1E8D0-U+1E8D6
"\240\158\165\132-\240\158\165\138") -- U+1E944-U+1E94A
 
-- Double combining characters.
-- Charset: [[:M:]&[:Canonical_Combining_Class=/^Double_/:]&[:^subhead=Grapheme joiner:]&[:^Variation_Selector=Yes:]]
local comb_chars_double =
"\205\156-\205\162" .. -- U+035C-U+0362
"\225\183\141" .. -- U+1DCD
"\225\183\188" -- U+1DFC
-- Variation selectors etc.; separated out so that we don't get categories for them.
-- Charset: [[:M:]&[[:subhead=Grapheme joiner:][:Variation_Selector=Yes:]]].
local comb_chars_other =
"\205\143" .. -- U+034F
"\225\160\139-\225\160\141" .. -- U+180B-U+180D
"\225\160\143" .. -- U+180F
"\239\184\128-\239\184\143" .. -- U+FE00-U+FE0F
"\243\160\132\128-\243\160\135\175" -- U+E0100-U+E01EF


local comb_chars_all = comb_chars.single .. comb_chars.double .. comb_chars.vs
local comb_chars_all = comb_chars_single .. comb_chars_double .. comb_chars_other


comb_chars = {
local comb_chars = {
combined_single = "[^" .. comb_chars_all .. "][" .. comb_chars.single .. comb_chars.vs .. "]+%f[^" .. comb_chars_all .. "]",
combined_single = "[^" .. comb_chars_all .. "][" .. comb_chars_single .. comb_chars_other .. "]+%f[^" .. comb_chars_all .. "]",
combined_double = "[^" .. comb_chars_all .. "][" .. comb_chars.single .. comb_chars.vs .. "]*[" .. comb_chars.double .. "]+[" .. comb_chars_all .. "]*.[" .. comb_chars.single .. comb_chars.vs .. "]*",
combined_double = "[^" .. comb_chars_all .. "][" .. comb_chars_single .. comb_chars_other .. "]*[" .. comb_chars_double .. "]+[" .. comb_chars_all .. "]*.[" .. comb_chars_single .. comb_chars_other .. "]*",
diacritics_single = "[" .. comb_chars.single .. "]",
diacritics_single = "[" .. comb_chars_single .. "]",
diacritics_double = "[" .. comb_chars.double .. "]",
diacritics_double = "[" .. comb_chars_double .. "]",
diacritics_all = "[" .. comb_chars_all .. "]"
diacritics_all = "[" .. comb_chars_all .. "]"
}
}


-- From https://unicode.org/Public/emoji/15.1/emoji-sequences.txt
-- Somewhat curated list from https://unicode.org/Public/emoji/16.0/emoji-sequences.txt.
local emoji_chars = {
-- NOTE: There are lots more emoji sequences involving non-emoji Plane 0 symbols followed by 0xFE0F, which we don't
{0x231A, 0x231B}, -- watch..hourglass done                                          # E0.6  [2] (⌚..⌛)
-- (yet?) handle.
{0x23E9, 0x23EC}, -- fast-forward button..fast down button                          # E0.6  [4] (⏩..⏬)
local emoji_chars =
0x23F0,          -- alarm clock                                                    # E0.6  [1] (⏰)
"\226\140\154" .. -- U+231A (⌚)
0x23F3,          -- hourglass not done                                            # E0.6  [1] (⏳)
"\226\140\155" .. -- U+231B (⌛)
{0x25FD, 0x25FE}, -- white medium-small square..black medium-small square          # E0.6  [2] (◽..◾)
"\226\140\168" .. -- U+2328 (⌨)
{0x2614, 0x2615}, -- umbrella with rain drops..hot beverage                        # E0.6  [2] (☔..☕)
"\226\143\143" .. -- U+23CF (⏏)
{0x2648, 0x2653}, -- Aries..Pisces                                                  # E0.6  [12] (♈..♓)
"\226\143\169-\226\143\179" .. -- U+23E9-U+23F3 (⏩-⏳)
0x267F,          -- wheelchair symbol                                              # E0.6  [1] (♿)
"\226\143\184-\226\143\186" .. -- U+23F8-U+23FA (⏸-⏺)
0x2693,          -- anchor                                                        # E0.6  [1] (⚓)
"\226\150\170" .. -- U+25AA (▪)
0x26A1,          -- high voltage                                                  # E0.6  [1] (⚡)
"\226\150\171" .. -- U+25AB (▫)
{0x26AA, 0x26AB}, -- white circle..black circle                                    # E0.6  [2] (⚪..⚫)
"\226\150\182" .. -- U+25B6 (▶)
{0x26BD, 0x26BE}, -- soccer ball..baseball                                          # E0.6  [2] (⚽..⚾)
"\226\151\128" .. -- U+25C0 (◀)
{0x26C4, 0x26C5}, -- snowman without snow..sun behind cloud                        # E0.6  [2] (⛄..⛅)
"\226\151\187-\226\151\190" .. -- U+25FB-U+25FE (◻-◾)
0x26CE,          -- Ophiuchus                                                      # E0.6  [1] (⛎)
"\226\152\128-\226\152\132" .. -- U+2600-U+2604 (☀-☄)
0x26D4,          -- no entry                                                      # E0.6  [1] (⛔)
"\226\152\142" .. -- U+260E (☎)
0x26EA,          -- church                                                        # E0.6  [1] (⛪)
"\226\152\145" .. -- U+2611 (☑)
{0x26F2, 0x26F3}, -- fountain..flag in hole                                        # E0.6  [2] (⛲..⛳)
"\226\152\148" .. -- U+2614 (☔)
0x26F5,          -- sailboat                                                      # E0.6  [1] (⛵)
"\226\152\149" .. -- U+2615 (☕)
0x26FA,          -- tent                                                          # E0.6  [1] (⛺)
"\226\152\152" .. -- U+2618 (☘)
0x26FD,          -- fuel pump                                                      # E0.6  [1] (⛽)
"\226\152\157" .. -- U+261D (☝)
0x2705,          -- check mark button                                              # E0.6  [1] (✅)
"\226\152\160" .. -- U+2620 (☠)
{0x270A, 0x270B}, -- raised fist..raised hand                                      # E0.6  [2] (✊..✋)
"\226\152\162" .. -- U+2622 (☢)
0x2728,          -- sparkles                                                      # E0.6  [1] (✨)
"\226\152\163" .. -- U+2623 (☣)
0x274C,          -- cross mark                                                    # E0.6  [1] (❌)
"\226\152\166" .. -- U+2626 (☦)
0x274E,          -- cross mark button                                              # E0.6  [1] (❎)
"\226\152\170" .. -- U+262A (☪)
{0x2753, 0x2755}, -- red question mark..white exclamation mark                      # E0.6  [3] (❓..❕)
"\226\152\174" .. -- U+262E (☮)
0x2757,          -- red exclamation mark                                          # E0.6  [1] (❗)
"\226\152\175" .. -- U+262F (☯)
{0x2795, 0x2797}, -- plus..divide                                                  # E0.6  [3] (➕..➗)
"\226\152\184-\226\152\186" .. -- U+2638-U+263A (☸-☺)
0x27B0,          -- curly loop                                                    # E0.6  [1] (➰)
"\226\153\136-\226\153\147" .. -- U+2648-U+2653 (♈-♓)
0x27BF,          -- double curly loop                                              # E1.0  [1] (➿)
"\226\153\159" .. -- U+265F (♟)
{0x2B1B, 0x2B1C}, -- black large square..white large square                        # E0.6  [2] (⬛..⬜)
"\226\153\160" .. -- U+2660 (♠)
0x2B50,          -- star                                                          # E0.6  [1] (⭐)
"\226\153\163" .. -- U+2663 (♣)
0x2B55,          -- hollow red circle                                              # E0.6  [1] (⭕)
"\226\153\165" .. -- U+2665 (♥)
{0x1F300, 0x1FAFF}, -- emoji in Plane 1
"\226\153\166" .. -- U+2666 (♦)
-- NOTE: There are lots more emoji sequences involving non-emoji Plane 0 symbols followed by 0xFE0F, which we don't
"\226\153\168" .. -- U+2668 (♨)
-- (yet?) handle.
"\226\153\187" .. -- U+267B (♻)
}
"\226\153\190" .. -- U+267E (♾)
emoji_chars = char_ranges_to_pattern(emoji_chars)
"\226\153\191" .. -- U+267F (♿)
"\226\154\146-\226\154\151" .. -- U+2692-U+2697 (⚒-⚗)
"\226\154\153" .. -- U+2699 (⚙)
"\226\154\155" .. -- U+269B (⚛)
"\226\154\156" .. -- U+269C (⚜)
"\226\154\160" .. -- U+26A0 (⚠)
"\226\154\161" .. -- U+26A1 (⚡)
"\226\154\170" .. -- U+26AA (⚪)
"\226\154\171" .. -- U+26AB (⚫)
"\226\154\176" .. -- U+26B0 (⚰)
"\226\154\177" .. -- U+26B1 (⚱)
"\226\154\189" .. -- U+26BD (⚽)
"\226\154\190" .. -- U+26BE (⚾)
"\226\155\132" .. -- U+26C4 (⛄)
"\226\155\133" .. -- U+26C5 (⛅)
"\226\155\136" .. -- U+26C8 (⛈)
"\226\155\142" .. -- U+26CE (⛎)
"\226\155\143" .. -- U+26CF (⛏)
"\226\155\145" .. -- U+26D1 (⛑)
"\226\155\147" .. -- U+26D3 (⛓)
"\226\155\148" .. -- U+26D4 (⛔)
"\226\155\169" .. -- U+26E9 (⛩)
"\226\155\170" .. -- U+26EA (⛪)
"\226\155\176-\226\155\181" .. -- U+26F0-U+26F5 (⛰-⛵)
"\226\155\183-\226\155\186" .. -- U+26F7-U+26FA (⛷-⛺)
"\226\155\189" .. -- U+26FD (⛽)
"\226\156\130" .. -- U+2702 (✂)
"\226\156\133" .. -- U+2705 (✅)
"\226\156\136-\226\156\141" .. -- U+2708-U+270D (✈-✍)
"\226\156\143" .. -- U+270F (✏)
"\226\156\146" .. -- U+2712 (✒)
"\226\156\148" .. -- U+2714 (✔)
"\226\156\150" .. -- U+2716 (✖)
"\226\156\157" .. -- U+271D (✝)
"\226\156\161" .. -- U+2721 (✡)
"\226\156\168" .. -- U+2728 (✨)
"\226\156\179" .. -- U+2733 (✳)
"\226\156\180" .. -- U+2734 (✴)
"\226\157\132" .. -- U+2744 (❄)
"\226\157\135" .. -- U+2747 (❇)
"\226\157\140" .. -- U+274C (❌)
"\226\157\142" .. -- U+274E (❎)
"\226\157\147-\226\157\149" .. -- U+2753-U+2755 (❓-❕)
"\226\157\151" .. -- U+2757 (❗)
"\226\157\163" .. -- U+2763 (❣)
"\226\157\164" .. -- U+2764 (❤)
"\226\158\149-\226\158\151" .. -- U+2795-U+2797 (➕-➗)
"\226\158\161" .. -- U+27A1 (➡)
"\226\158\176" .. -- U+27B0 (➰)
"\226\158\191" .. -- U+27BF (➿)
"\226\164\180" .. -- U+2934 (⤴)
"\226\164\181" .. -- U+2935 (⤵)
"\226\172\133-\226\172\135" .. -- U+2B05-U+2B07 (⬅-⬇)
"\226\172\155" .. -- U+2B1B (⬛)
"\226\172\156" .. -- U+2B1C (⬜)
"\226\173\144" .. -- U+2B50 (⭐)
"\226\173\149" .. -- U+2B55 (⭕)
"\227\128\176" .. -- U+3030 (〰)
"\227\128\189" .. -- U+303D (〽)
"\227\138\151" .. -- U+3297 (㊗)
"\227\138\153" .. -- U+3299 (㊙)
"\240\159\128\132" .. -- U+1F004 (🀄)
"\240\159\131\143" .. -- U+1F0CF (🃏)
"\240\159\133\176" .. -- U+1F170 (🅰)
"\240\159\133\177" .. -- U+1F171 (🅱)
"\240\159\133\190" .. -- U+1F17E (🅾)
"\240\159\133\191" .. -- U+1F17F (🅿)
"\240\159\134\142" .. -- U+1F18E (🆎)
"\240\159\134\145-\240\159\134\154" .. -- U+1F191-U+1F19A (🆑-🆚)
"\240\159\136\129" .. -- U+1F201 (🈁)
"\240\159\136\130" .. -- U+1F202 (🈂)
"\240\159\136\154" .. -- U+1F21A (🈚)
"\240\159\136\175" .. -- U+1F22F (🈯)
"\240\159\136\178-\240\159\136\186" .. -- U+1F232-U+1F23A (🈲-🈺)
"\240\159\137\144" .. -- U+1F250 (🉐)
"\240\159\137\145" .. -- U+1F251 (🉑)
"\240\159\140\128-\240\159\153\143" .. -- U+1F300-U+1F64F (🌀-🙏)
"\240\159\154\128-\240\159\155\151" .. -- U+1F680-U+1F6D7 (🚀-🛗)
"\240\159\155\156-\240\159\155\172" .. -- U+1F6DC-U+1F6EC (🛜-🛬)
"\240\159\155\176-\240\159\155\188" .. -- U+1F6F0-U+1F6FC (🛰-🛼)
"\240\159\159\160-\240\159\159\171" .. -- U+1F7E0-U+1F7EB (🟠-🟫)
"\240\159\159\176" .. -- U+1F7F0 (🟰)
"\240\159\164\140-\240\159\169\147" .. -- U+1F90C-U+1FA53 (🤌-🩓)
"\240\159\169\160-\240\159\169\173" .. -- U+1FA60-U+1FA6D (🩠-🩭)
"\240\159\169\176-\240\159\169\188" .. -- U+1FA70-U+1FA7C (🩰-🩼)
"\240\159\170\128-\240\159\170\137" .. -- U+1FA80-U+1FA89 (🪀-🪉)
"\240\159\170\143-\240\159\171\134" .. -- U+1FA8F-U+1FAC6 (🪏-🫆)
"\240\159\171\142-\240\159\171\156" .. -- U+1FACE-U+1FADC (🫎-🫜)
"\240\159\171\159-\240\159\171\169" .. -- U+1FADF-U+1FAE9 (🫟-🫩)
"\240\159\171\176-\240\159\171\184" -- U+1FAF0-U+1FAF8 (🫰-🫸)


local unsupported_characters = {}
local unsupported_characters
for k, v in pairs(require("Module:links/data").unsupported_characters) do
local function get_unsupported_characters()
unsupported_characters[v] = k
unsupported_characters, get_unsupported_characters = {}, nil
for k, v in pairs(load_data("Module:links/data").unsupported_characters) do
unsupported_characters[v] = k
end
return unsupported_characters
end
end


-- Get the list of unsupported titles and invert it (so the keys are pagenames and values are canonical titles).
-- The list of unsupported titles and invert it (so the keys are pagenames and values are canonical titles).
local unsupported_titles = {}
local unsupported_titles
for k, v in pairs(require("Module:links/data").unsupported_titles) do
local function get_unsupported_titles()
unsupported_titles[v] = k
unsupported_titles, get_unsupported_titles = {}, nil
for k, v in pairs(load_data("Module:links/data").unsupported_titles) do
unsupported_titles[v] = k
end
return unsupported_titles
end
end


Line 466: Line 671:
   with {mw.ustring.find()}.
   with {mw.ustring.find()}.
* `L2_list`: Ordered list of L2 headings on the page, with the extra key `n` that gives the length of the list.
* `L2_list`: Ordered list of L2 headings on the page, with the extra key `n` that gives the length of the list.
* `L2_sections`: Lookup table of L2 headings on the page, where the key is the section number assigned by the preprocessor, and the value is the L2 heading name. Once an invocation has got its actual section number from get_current_section in [[Module:utilities]], it can use this table to determine its parent L2. TODO: We could expand this to include subsections, to check POS headings are correct etc.
* `L2_sections`: Lookup table of L2 headings on the page, where the key is the section number assigned by the preprocessor, and the value is the L2 heading name. Once an invocation has got its actual section number from get_current_L2 in [[Module:pages]], it can use this table to determine its parent L2. TODO: We could expand this to include subsections, to check POS headings are correct etc.
* `unsupported_titles`: Map from pagenames to canonical titles for unsupported-title pages.
* `unsupported_titles`: Map from pagenames to canonical titles for unsupported-title pages.
* `namespace`: Namespace of the pagename.
* `namespace`: Namespace of the pagename.
Line 489: Line 694:
comb_chars = comb_chars,
comb_chars = comb_chars,
emoji_pattern = "[" .. emoji_chars .. "]",
emoji_pattern = "[" .. emoji_chars .. "]",
unsupported_titles = unsupported_titles
unsupported_titles = unsupported_titles or get_unsupported_titles()
}
}
Line 501: Line 706:
error("Internal error: Something wrong, `data.pagename` not specified but current title containg illegal characters")
error("Internal error: Something wrong, `data.pagename` not specified but current title containg illegal characters")
else
else
error(("Bad value for `data.pagename`: '%s', which must not contain illegal characters"):format(pagename))
error(format("Bad value for `data.pagename`: '%s', which must not contain illegal characters", pagename))
end
end
end
end
Line 517: Line 722:


local frame = mw.getCurrentFrame()
local frame = mw.getCurrentFrame()
-- WARNING: `content` May be nil, e.g. if we're substing a template like {{ja-new}} on a not-yet-created page
-- WARNING: `content` may be nil, e.g. if we're substing a template like {{ja-new}} on a not-yet-created page
-- or if the module specifies the subpage as `data.pagename` (which many modules do) and we're in an Appendix
-- or if the module specifies the subpage as `data.pagename` (which many modules do) and we're in an Appendix
-- or other non-mainspace page. We used to make the latter an error but there are too many modules that do it,
-- or other non-mainspace page. We used to make the latter an error but there are too many modules that do it,
Line 523: Line 728:
-- content of the page.
-- content of the page.
local content = raw_title:getContent()
local content = raw_title:getContent()
local content_lang = mw.getContentLanguage()


--Get the pagename.
-- Get the pagename.
pagename = raw_title.subpageText
pagename = gsub(raw_title.subpageText, "^Unsupported titles/(.+)", function(m)
:gsub("^Unsupported titles/(.*)", function(m)
insert(cats, "Unsupported titles")
insert(cats, "Unsupported titles")
local title = (unsupported_titles or get_unsupported_titles())[m]
return unsupported_titles[m] or (m:gsub("`.-`", unsupported_characters))
if title then
end)
return title
end
-- Substitute pairs of "`". Those not used for escaping should be escaped as "`grave`", but might not be,
-- so if a pair don't form a match, the closing "`" should become the opening "`" of the next match attempt.
-- This has to be done manually, instead of using gsub.
local open_pos = find(m, "`")
if not open_pos then
return m
end
title = {sub(m, 1, open_pos - 1)}
while true do
local close_pos = find(m, "`", open_pos + 1)
if not close_pos then
-- Add "`" plus any remaining characters.
insert(title, sub(m, open_pos))
break
end
local escape = sub(m, open_pos, close_pos)
local ch = (unsupported_characters or get_unsupported_characters())[escape]
-- Match found, so substitute the character and move to the first "`" after the match if found, or
-- otherwise return.
if ch then
insert(title, ch)
local nxt_pos = close_pos + 1
open_pos = find(m, "`", nxt_pos)
-- Add any characters between the match and the next "`" or end.
if open_pos then
insert(title, sub(m, nxt_pos, open_pos - 1))
else
insert(title, sub(m, nxt_pos))
break
end
-- Match not found, so make the closing "`" the opening "`" of the next attempt.
else
-- Add the failed match, except for the closing "`".
insert(title, sub(m, open_pos, close_pos - 1))
open_pos = close_pos
end
end
return concat(title)
end)
-- Save pagename, as local variable will be destructively modified.
-- Save pagename, as local variable will be destructively modified.
data.pagename = pagename
data.pagename = pagename
Line 544: Line 789:
end
end
pagename = ugsub(pagename, comb_chars.combined_double, explode)
pagename = ugsub(pagename, comb_chars.combined_double, explode)
pagename = ugsub(pagename, comb_chars.combined_single, explode)
pagename = gsub(ugsub(pagename, comb_chars.combined_single, explode), ".[\128-\191]*", explode)
:gsub(".[\128-\191]*", explode)


data.explode_pagename = explode_pagename
data.explode_pagename = explode_pagename
Line 552: Line 796:
-- Generate DEFAULTSORT.
-- Generate DEFAULTSORT.
data.encoded_pagename = encode_entities(data.pagename)
data.encoded_pagename = encode_entities(data.pagename)
data.pagename_defaultsort = require("Module:languages").getByCode("mul"):makeSortKey(data.encoded_pagename)
data.pagename_defaultsort = get_lang("mul"):makeSortKey(data.encoded_pagename)
frame:callParserFunction(
frame:callParserFunction("DEFAULTSORT", data.pagename_defaultsort)
"DEFAULTSORT",
data.raw_defaultsort = uupper(raw_title.text)
data.pagename_defaultsort
)
data.raw_defaultsort = raw_title.text:uupper()
-- Make `L2_list` and `L2_sections`, note raw wikitext use of {{DEFAULTSORT:}} and {{DISPLAYTITLE:}}, then add categories if any unwanted L1 headings are found, the L2 headings are in the wrong order, or they don't match a canonical language name.
-- Make `L2_list` and `L2_sections`, note raw wikitext use of {{DEFAULTSORT:}} and {{DISPLAYTITLE:}}, then add categories if any unwanted L1 headings are found, the L2 headings are in the wrong order, or they don't match a canonical language name.
Line 563: Line 804:
do
do
local L2_list, L2_list_len, L2_sections, sort_cache, prev = {}, 0, {}, {}
local L2_list, L2_list_len, L2_sections, sort_cache, prev = {}, 0, {}, {}
local defaultsort, displaytitle, page_has_L1, L2_wrong_order, L2_nonstandard, page_has_arg
local new_cats, L2_wrong_order = {}
local function get_weight(L2)
local function get_weight(L2)
Line 582: Line 823:
end
end
local function handle_heading(node)
local function handle_heading(heading)
local level = node.level
local level = heading.level
if level > 2 then
if level > 2 then
return
return
end
end
local name = node:get_name()
local name = heading:get_name()
-- Check there are no newline characters in the heading, which might appear after preprocessing (e.g. from an expanded template). In such cases, the preprocessor section count still increments (since it's calculated pre-expansion), but the heading will fail, so we shouldn't increment the L2 count.
-- heading:get_name() will return nil if there are any newline characters in the preprocessed heading name (e.g. from an expanded template). In such cases, the preprocessor section count still increments (since it's calculated pre-expansion), but the heading will fail, so the L2 count shouldn't be incremented.
if find(name, "\n", 1, true) then
if name == nil then
return
return
end
end
L2_list_len = L2_list_len + 1
L2_list_len = L2_list_len + 1
L2_list[L2_list_len] = name
L2_list[L2_list_len] = name
L2_sections[node.section] = name
L2_sections[heading.section] = name
-- We also add any L1s, since they terminate the preceding L2, but add a maintenance category since it's probably a mistake.
-- Also add any L1s, since they terminate the preceding L2, but add a maintenance category since it's probably a mistake.
if level == 1 then
if level == 1 then
page_has_L1 = true
new_cats["Pages with unwanted L1 headings"] = true
end
end
-- Check the heading is in the right order.
-- Check the heading is in the right order.
Line 605: Line 846:
string_sort(get_weight(prev), get_weight(name))
string_sort(get_weight(prev), get_weight(name))
) then
) then
new_cats["Pages with language headings in the wrong order"] = true
L2_wrong_order = true
L2_wrong_order = true
end
end
-- Check it's a canonical language name.
-- Check it's a canonical language name.
if not langnames[name] then
if not (langnames or get_langnames())[name] then
L2_nonstandard = true
new_cats["Pages with nonstandard language headings"] = true
end
end
prev = name
prev = name
end
end
local function handle_template(node)
local function handle_template(template)
local name = node:get_name()
local name = template:get_name()
if name == "DEFAULTSORT:" then
if name == "DEFAULTSORT:" then
defaultsort = true
new_cats["Pages with DEFAULTSORT conflicts"] = true
elseif name == "DISPLAYTITLE:" then
elseif name == "DISPLAYTITLE:" then
displaytitle = true
new_cats["Pages with DISPLAYTITLE conflicts"] = true
end
end
end
end
if content then
if content then
for node in require("Module:template parser").parse(content):__pairs("next_node") do
for node in parse(content):iterate_nodes() do
local node_type = type_or_class(node)
local node_class = class_else_type(node)
if node_type == "heading" then
if node_class == "heading" then
handle_heading(node)
handle_heading(node)
elseif node_type == "template" and not (defaultsort and displaytitle) then
elseif node_class == "template" then
handle_template(node)
handle_template(node)
elseif node_type == "argument" then
elseif node_class == "parameter" then
page_has_arg = true
new_cats["Pages with raw triple-brace template parameters"] = true
end
end
end
end
Line 639: Line 881:
data.L2_list = L2_list
data.L2_list = L2_list
data.L2_sections = L2_sections
data.L2_sections = L2_sections
 
insert(cats, get_category("Pages with entries"))
insert(cats, get_category(format("Pages with %s entr%s", L2_list_len, L2_list_len == 1 and "y" or "ies")))
for cat in pairs(new_cats) do
insert(cats, get_category(cat))
end
end
end


Line 676: Line 924:
end
end
t_lang[uupper(decode_entities(sortkey))] = true
t_lang[uupper(decode_entities(sortkey))] = true
end
local function do_iteration(name, sortkey, wikitext_langname_cat)
if langnames[name] then
return add_cat_table(wikitext_langname_cat, name, sortkey)
end
name = etym_langnames[name] and name or content_lang:lcfirst(name)
if etym_langnames[name] then
name = get_etym_lang(name):getFullName()
return add_cat_table(wikitext_langname_cat, name, sortkey)
end
end
end
Line 711: Line 948:
local code = match(cat, "^([%w%-.]+):")
local code = match(cat, "^([%w%-.]+):")
if code then
if code then
return add_cat_table(wikitext_topic_cat, code, sortkey)
add_cat_table(wikitext_topic_cat, code, sortkey)
return
end
end
-- Split by word.
-- Split by word.
cat = split(cat, " ", true, true)
cat = split(cat, " ", true, true)
-- Iterate over the category name, starting with the longest possible name and shaving off the first word until we find one. We do it this way because:
-- Formerly we looked for the language name anywhere in the category. This is simply wrong
-- (a) Going from shortest to longest risks falsely matching (e.g.) German Low German categories as German.
-- because there are no categories like 'Alsatian French lemmas' (only L2 languages
-- (b) Checking the start of category names first risks falsely match (e.g.) Alsatian French as Alsatian (a variety of Alemannic German), not French.
-- have langname categories), but doing it this way wrongly catches things like [[Category:Shapsug Adyghe]]
-- If no matches are found, then check the start of the category name, shaving off the last word each iteration.
-- in [[Category:Adyghe entries with language name categories using raw markup]].
local cat_len, n, name, done = #cat, 1
local n = #cat - 1
repeat
name = concat(cat, " ", n, cat_len)
done = do_iteration(name, sortkey, wikitext_langname_cat)
if done then
return
end
n = n + 1
until n > cat_len
n = cat_len - 1
if n <= 0 then
if n <= 0 then
return
return
end
end
-- Go from longest to shortest and stop once we've found a language name. Going from shortest
-- to longest or not stopping after a match risks falsely matching (e.g.) German Low German
-- categories as German.
repeat
repeat
name = concat(cat, " ", 1, n)
local name = concat(cat, " ", 1, n)
done = do_iteration(name, sortkey, wikitext_langname_cat)
if (langnames or get_langnames())[name] then
if done then
add_cat_table(wikitext_langname_cat, name, sortkey)
return
return
end
end
Line 769: Line 1,001:
data.wikitext_topic_cat = wikitext_topic_cat
data.wikitext_topic_cat = wikitext_topic_cat
data.wikitext_langname_cat = wikitext_langname_cat
data.wikitext_langname_cat = wikitext_langname_cat
if raw_sortkey then
insert(cats, get_category("Pages with raw sortkeys"))
end
end
end


Retrieved from "https://duo.linguifex.com/wiki/Module:headword/page"