module djinn.parsing;

@safe:

import std.algorithm;
import std.array;
import std.exception;
import std.range;
import std..string;
import std.utf;

import djinn.lexing;
import djinn.types;

package:

/// Turns Djinn code into post-processed tokens (entry point to all Djinn parsing)
Token[] getTokens(Source* src) pure
{
	if (src.empty) return Token[].init;
	auto tokens_app = appender!(Token[]);
	parseTokens(src, tokens_app);
	auto tokens = tokens_app[];
	return trimWhitespace(tokens, src.line_offsets);
}

private:

/**
  Post-processes an array of tokens to implement whitespace-trimming rules

  These rules allow using newlines, etc., to format Djinn code, without affecting the output
*/
Token[] trimWhitespace(Token[] tokens, const(size_t[]) line_offsets) pure
{
	// Djinn's whitespace rule: we don't render lines that contain at least some [< or {% tokens and nothing else but whitespace
	// This loop scans each line and checks the tokens intersecting them in total time O(num_lines + num_tokens), trimming whitespace from text tokens if the rule is triggered
	size_t line_start_token_idx = 0;
	foreach (j; 0..line_offsets.length-1)
	{
		const line_offset = line_offsets[j];
		const line_end_offset = line_offsets[j+1];

		while (tokens[line_start_token_idx+1].pos.offset <= line_offset)
		{
			line_start_token_idx++;
		}

		size_t line_end_token_idx = line_start_token_idx+1;
		bool has_exprs = tokens[line_start_token_idx].type == TokenType.expressions, is_all_text = tokens[line_start_token_idx].type == TokenType.text;
		while (tokens[line_end_token_idx].pos.offset < line_end_offset)
		{
			has_exprs |= tokens[line_end_token_idx].type == TokenType.expressions;
			is_all_text &= tokens[line_end_token_idx].type == TokenType.text;
			line_end_token_idx++;
		}

		if (has_exprs || is_all_text)
		{
			line_start_token_idx = line_end_token_idx - 1;
			continue;
		}

		bool rendersWhiteOnLine(ref const(Token) token)
		{
			assert (token.type != TokenType.expressions);
			if (token.type != TokenType.text) return true;
			import std.uni : isWhite;
			string value = token.value;
			if (token.pos.offset < line_offset) value = value[line_offset-token.pos.offset..$];
			if (token.pos.offset + token.value.length > line_end_offset) value = value[0..$-(token.pos.offset+token.value.length-line_end_offset)];
			return value.all!(isWhite);
		}
		if (!tokens[line_start_token_idx..line_end_token_idx].all!rendersWhiteOnLine) continue;

		if (tokens[line_start_token_idx].type == TokenType.text)
		{
			const t = tokens[line_start_token_idx];
			tokens[line_start_token_idx].value = t.value[0..line_offset-t.pos.offset];
		}
		foreach (k; line_start_token_idx+1..line_end_token_idx-1) if (tokens[k].type == TokenType.text) tokens[k].value = "";
		if (tokens[line_end_token_idx-1].type == TokenType.text)
		{
			const t = tokens[line_end_token_idx-1];
			tokens[line_end_token_idx-1].value = t.value[line_end_offset-t.pos.offset..$];
		}

		line_start_token_idx = line_end_token_idx - 1;
	}

	// This handles the [:- and -:] whitespace-stripping feature, as in Jinja2
	foreach (j; 0..tokens.length-1)
	{
		// FIXME: correct pos
		if (tokens[j].whitespace_stripping & WhitespaceStripping.right) tokens[j+1].value = stripLeft(tokens[j+1].value);
		if (tokens[j+1].whitespace_stripping & WhitespaceStripping.left) tokens[j].value = stripRight(tokens[j].value);
	}
	return tokens;
}

/// Parses complete Djinn code, outputting tokens
void parseTokens(Source* src, ref TokenSink output) pure
{
	while (!src.empty)
	{
		foreach (j; 0..src.rem.length-1)
		{
			if (src.rem[j] == '[')
			{
				auto type = cast(TokenType)src.rem[j+1].among(':', '=', '<');
				if (type == TokenType.text) continue;

				if (j > 0) output.put(Token(TokenType.text, src.rem[0..j].source, src.curPos()));
				src.rem = src.rem[j..$];
				const pos = src.curPos();
				src.rem = src.rem[2..$];

				with (TokenType) final switch (type)
				{
					case text:
						assert (false);

					case statements:
						output.put(parseD(src, type, pos));
						break;

					case expressions:
						output.put(parseD(src, type, pos));
						break;

					case directive:
						parseDirective(src, pos, output);
						break;
				}

				goto next;
			}
		}
		output.put(Token(TokenType.text, src.rem.source, src.curPos()));
		src.rem = src.rem[$..$];
next:
	}
	output.put(Token(TokenType.directive, "EOF", src.curPos()));
}

/// Parses something like "dCode(42); :]" when terminator == ":]"
Token parseD(Source* src, TokenType type, ref const(Pos) pos) pure
{
	assert (type.among(TokenType.statements, TokenType.expressions));
	/*
		Expressions must be fully balanced.  We need to check this to disambiguate the ] terminator.
		Statements don't need to be balanced.  E.g. (contrived example):
			lorem ipsum
			[: processThunks([{ :]
				Here's a lambda in an array passed to a function call.
			[: }]); :]
	*/
	Nestables balance_what = kAllNestables;
	string terminator = "]";
	if (type == TokenType.statements)
	{
		balance_what = Nestables.init;
		terminator = ":]";
	}
	try
	{
		auto rem = dSkipFind(src.rem, terminator.byCodeUnit, balance_what, src.offset);
		const value = src.rem[0..src.rem.length-rem.length];
		src.rem = rem[terminator.length..$];
		return makeCodeToken(type, value, pos);
	}
	catch (LexicalException e)
	{
		const e_pos = src.posAt(e.offset);
		throw syntaxException(e.msg, e_pos);
	}
}

pure
unittest
{
	bool test(string code, TokenType type, Token exp) pure
	{
		auto src = new Source("foo.dj", code);
		const pos = src.curPos();
		auto result = parseD(src, type, pos);
		exp.pos = result.pos;  // Ignore pos for this test
		return result == exp;
	}

	with (TokenType)
	{
		assert (test("dCode(42); :]", statements, Token(statements, "dCode(42); ")));
		assert (test("writeln(`:]`); :]", statements, Token(statements, "writeln(`:]`); ")));
		assert (test("processThunks([{ :]", statements, Token(statements, "processThunks([{ ")));
		assert (test(":]", statements, Token(statements, "")));
		assertThrown(test("]]", statements, Token.init));
		assertThrown(test("", statements, Token.init));

		assert (test("dCode(42) ]", expressions, Token(expressions, "dCode(42) ")));
		assert (test("1, 2, 3]", expressions, Token(expressions, "1, 2, 3")));
		assert (test("[]]", expressions, Token(expressions, "[]")));
		assert (test("`]` ]", expressions, Token(expressions, "`]` ")));
		assert (test(`q"[[]]"]`, expressions, Token(expressions, `q"[[]]"`)));
		assertThrown(test("processThunks([{ ]", expressions, Token.init));
		assert (test("]", expressions, Token(expressions, "")));
	}
}

/// Parses something like "foo bar >]"
void parseDirective(Source* src, ref const(Pos) pos, ref TokenSink output) pure
{
	import std.conv : text;
	auto pieces = src.rem.findSplit(">]".byCodeUnit);
	if (!pieces) throw syntaxException("no >] terminator found", pos);
	src.rem = pieces[2];
	auto directive = makeCodeToken(TokenType.directive, pieces[0], pos);
	output.put(directive);
	auto args = directive.value.splitter.filter!(e => !e.empty);
	if (args.empty || args.front != "raw")
	{
		return;
	}
	args.popFront();

	string8b end_tag;
	if (!args.empty)
	{
		static import std.ascii;
		end_tag = args.front.byCodeUnit;
		enforce(end_tag.all!(c => std.ascii.isAlphaNum(c) || c == '_'), syntaxException(text("raw directive ending tag must be identifier using ASCII letters, numbers and underscores (got ", end_tag, ")"), src));
		args.popFront();
		enforce(args.empty, syntaxException("extra arguments for raw directive (maximum 1 supported)", src));
	}

	// Search for [< endraw TAG !}
	foreach (j; 0..src.rem.length)
	{
		auto match = endRawMatch(src.rem[j..$], end_tag);
		if (match.empty) continue;
		const raw_text_pos = src.curPos();
		output.put(makeCodeToken(TokenType.text, src.rem[0..j], raw_text_pos));
		src.rem = src.rem[j..$];
		const end_tag_pos = src.curPos();
		output.put(makeCodeToken(TokenType.directive, match[2..$-2], end_tag_pos));
		src.rem = src.rem[match.length..$];
		return;
	}

	throw syntaxException(text("no [< endraw", end_tag.empty ? "" : " ", end_tag, " >] terminator found"), src);
}

unittest
{
	bool test(string code, Token[] exp) @trusted
	{
		TokenSink app;
		auto src = new Source("foo.dj", code);
		const pos = src.curPos();
		parseDirective(src, pos, app);
		if (app[].length != exp.length) return false;
		foreach (r, e; lockstep(app[], exp))
		{
			r.pos = e.pos;  // Ignore pos for this test
			if (r != e) return false;
		}
		return true;
	}

	assertThrown(test("", []));

	assert (test("foo bar >]", [Token(TokenType.directive, "foo bar ")]));
	assert (test(">]", [Token(TokenType.directive, "")]));
	assert (test("foo|>] lorem ipsum", [Token(TokenType.directive, "foo", Pos.init, WhitespaceStripping.right)]));
	assert (test("| foo >]", [Token(TokenType.directive, " foo ", Pos.init, WhitespaceStripping.left)]));
	assert (test("| foo |>]", [Token(TokenType.directive, " foo ", Pos.init, WhitespaceStripping.both)]));

	assert (test("raw>][<endraw>]lorem ipsum", [
		Token(TokenType.directive, "raw"),
		Token(TokenType.text, ""),
		Token(TokenType.directive, "endraw"),
	]));

	assert (test("| raw >]lorem ipsum[< endraw |>]", [
		Token(TokenType.directive, " raw ", Pos.init, WhitespaceStripping.left),
		Token(TokenType.text, "lorem ipsum"),
		Token(TokenType.directive, " endraw ", Pos.init, WhitespaceStripping.right),
	]));

	assert (test("raw|>]lorem [< ipsum[<|endraw>]", [
		Token(TokenType.directive, "raw", Pos.init, WhitespaceStripping.right),
		Token(TokenType.text, "lorem [< ipsum"),
		Token(TokenType.directive, "endraw", Pos.init, WhitespaceStripping.left),
	]));

	assert (test("raw tag>][< endraw >][<endraw tag>]", [
		Token(TokenType.directive, "raw tag"),
		Token(TokenType.text, "[< endraw >]"),
		Token(TokenType.directive, "endraw tag"),
	]));

	assert (test("raw tag >][< endraw >][<endraw tag>]", [
		Token(TokenType.directive, "raw tag "),
		Token(TokenType.text, "[< endraw >]"),
		Token(TokenType.directive, "endraw tag"),
	]));

	assertThrown(test("raw>]", []));
	assertThrown(test("raw tag>][<endraw>]", []));
}

/**
  Matches an endraw directive (with possible tag) at start of str

	Returns: matching substring, or empty if not matched
*/
string8b endRawMatch(string8b str, string8b tag) pure
{
	// Equivalent to regex `^[<|?\s*endraw\s*TAG\s*|?>]`
	enum miss = string8b.init;
	auto s = str;
	if (!s.startsWith("[<")) return miss;
	s = s[2..$];
	if (s.empty) return miss;
	if (s.front == '|') s.popFront();
	s = s.source.stripLeft.byCodeUnit;
	if (!s.startsWith("endraw")) return miss;
	s = s["endraw".length..$];
	s = s.source.stripLeft.byCodeUnit;
	if (!s.startsWith(tag)) return miss;
	s = s[tag.length..$];
	s = s.source.stripLeft.byCodeUnit;
	if (s.empty) return miss;
	if (s.front == '|') s.popFront();
	if (!s.startsWith(">]")) return miss;
	s = s[2..$];
	return str[0..str.length-s.length];
}

pure
unittest
{
	bool test(string str, string tag, string result)
	{
		return endRawMatch(str.byCodeUnit, tag.byCodeUnit) == result.byCodeUnit;
	}

	assert (test("", "", ""));
	assert (test("nope", "", ""));
	assert (test("", "nope", ""));
	assert (test("nope", "nope", ""));

	assert (test("[<endraw>]", "", "[<endraw>]"));
	assert (test("[< endraw >]", "", "[< endraw >]"));
	assert (test("[<  endraw  >]", "", "[<  endraw  >]"));
	assert (test("[< endraw >] lorem ipsum", "", "[< endraw >]"));
	assert (test("[<| endraw |>] lorem ipsum", "", "[<| endraw |>]"));
	assert (test("[< endraw >] lorem ipsum", "tag", ""));
	assert (test("[< endraw tag >] lorem ipsum", "tag", "[< endraw tag >]"));
	assert (test("[<endraw tag>] lorem ipsum", "tag", "[<endraw tag>]"));
}

Token makeCodeToken(TokenType type, string8b code, ref const(Pos) pos) pure
{
	WhitespaceStripping ws_strip;
	if (code.startsWith("|"))
	{
		ws_strip |= WhitespaceStripping.left;
		code = code[1..$];
	}
	if (code.endsWith("|"))
	{
		ws_strip |= WhitespaceStripping.right;
		code = code[0..$-1];
	}
	return Token(type, code.source, pos, ws_strip);
}