Module:User:Theknightwho/parser/tokens

This module sandbox lacks a documentation subpage. Please create it.
Useful links: root page • root page’s subpages • links • transclusions • testcases • sandbox
local error = error
local ipairs = ipairs
local pairs = pairs
local setmetatable = setmetatable

local table = table
local concat = table.concat
local insert = table.insert

local function __eq(a, b)
	if (
		a:get_type() == b:get_type() and
		a:get_size() == b:get_size()
	) then
		for k, v in pairs(a) do
			if b ~= v then
				return false
			end
		end
		return true
	end
	return false
end

local function __newindex()
	error("Not allowed to add extra keys to tokens.")
end

local function __tostring(t)
	local t_type = t:get_type()
	if #t == 0 then
		return t_type
	elseif t_type == "Text" then
		return t_type .. " \"" .. t .. "\""
	end
	local vals = {}
	for _, v in ipairs(t) do
		insert(vals, v)
	end
	return t_type .. " (" .. concat(vals, ",") .. ")"
end

local Token = {}

function Token:new(t)
	t = t or {}
	local size = 0
	for _ in pairs(t) do
		size = size + 1
	end
	t._size = size
	return setmetatable(t, {
		__eq = __eq,
		__index = self,
		__newindex = __newindex,
		__tostring = __tostring
	})
end

function Token:get_class()
	return self._class
end

function Token:get_size()
	return self._size
end

function Token:get_type()
	return self._type
end

local Tokens = {}
for _, v in ipairs{
	"Text",
	"TemplateOpen", -- {{
	"TemplateParamSeparator", -- |
	"TemplateParamEquals", -- =
	"TemplateClose", -- }}
	"ArgumentOpen", -- {{{
	"ArgumentSeparator", -- |
	"ArgumentClose", -- }}}
	"StripMarker", -- \127'"`UNIQ--tagname-XXXXXXXX-QINU`"'\127
	"WikilinkOpen", -- [[
	"WikilinkColonTrick", -- :
	"WikilinkInterwikiSeparator", -- :
	"WikilinkNamespaceSeparator", -- :
	"FragmentSeparator", -- #
	"WikilinkPipe", -- |
	"WikilinkClose", -- ]]
	"FileOpen", -- [[
	"FileSeparator", -- :
	"FilePipe", -- |
	"FileClose", -- ]]
	"BracketedExternalLinkOpen", -- [
	"BracketedExternalLinkSeparator", -- " "
	"BracketedExternalLinkClose", -- ]
	"FreeExternalLinkOpen",
	"FreeExternalLinkClose",
	"MagicLinkOpen",
	"MagicLinkSeparator",
	"MagicLinkClose",
	"MagicWordOpen", -- __
	"MagicWordClose", -- __
	"HeadingStart", -- =...
	"HeadingEnd", -- ...=
	"IndentationMarker", -- :
	"OrderedListMarker", -- #
	"UnorderedListMarker", -- *
	"DescriptionListMarker", -- ;
	"DescriptionListSeparator", -- :
	"StyleApostrophes2", -- ''
	"StyleApostrophes3", -- '''
	"StyleApostrophes5", -- '''''
	"ItalicsOpen", -- ''
	"ItalicsClose", -- ''
	"BoldOpen", -- '''
	"BoldClose", -- '''
	"HorizontalRule", -- ----
	"ParserTagOpen",
	"ParserTagClose",
	"ParserTagSelfClose",
	"Ignored",
	"TagOpen", -- <...>
	"TagClose", -- </...>
	"TagSelfClose" -- <.../>
} do
	local t = {_type = v}
	t._class = t
	Tokens = Token:new(t)
end

return Tokens
Module:User:Theknightwho/parser/tokens

Wikious

Boobota

Sagapedia
