/*
 * Copyright 2018 Richard Hughes <richard@hughsie.com>
 *
 * SPDX-License-Identifier: LGPL-2.1-or-later
 */

#define G_LOG_DOMAIN "XbSilo"

#include "config.h"

#include <gio/gio.h>
#include <string.h>

#include "xb-builder-fixup-private.h"
#include "xb-builder-node-private.h"
#include "xb-builder-source-private.h"
#include "xb-builder.h"
#include "xb-opcode-private.h"
#include "xb-silo-private.h"
#include "xb-string-private.h"
#include "xb-version.h"

#define XB_BUILDER_MAX_DEPTH 100

typedef struct {
	GPtrArray *sources; /* of XbBuilderSource */
	GPtrArray *nodes;   /* of XbBuilderNode */
	GPtrArray *fixups;  /* of XbBuilderFixup */
	GPtrArray *locales; /* of str */
	XbSiloProfileFlags profile_flags;
	GString *guid;
} XbBuilderPrivate;

G_DEFINE_TYPE_WITH_PRIVATE(XbBuilder, xb_builder, G_TYPE_OBJECT)
#define GET_PRIVATE(o) (xb_builder_get_instance_private(o))

typedef struct {
	XbSilo *silo;
	XbBuilderNode *root;	/* transfer full */
	XbBuilderNode *current; /* transfer none */
	XbBuilderCompileFlags compile_flags;
	XbBuilderSourceFlags source_flags;
	GHashTable *strtab_hash;
	GByteArray *strtab;
	GPtrArray *locales;
	gboolean elem_closed;
	guint depth;
	GError *error;
} XbBuilderCompileHelper;

static guint32
xb_builder_compile_add_to_strtab(XbBuilderCompileHelper *helper, const gchar *str)
{
	gpointer val;
	guint32 idx;
	gsize str_len;

	/* already exists */
	if (g_hash_table_lookup_extended(helper->strtab_hash, str, NULL, &val))
		return GPOINTER_TO_UINT(val);

	/* check for integer truncation */
	if (helper->strtab->len > G_MAXINT32) {
		g_set_error(&helper->error,
			    G_IO_ERROR,
			    G_IO_ERROR_INVALID_DATA,
			    "string table too large");
		return XB_SILO_UNSET;
	}

	/* check if adding this string would exceed the limit */
	str_len = strlen(str) + 1;
	if (helper->strtab->len + str_len > G_MAXUINT32) {
		g_set_error(&helper->error,
			    G_IO_ERROR,
			    G_IO_ERROR_INVALID_DATA,
			    "string table too large");
		return XB_SILO_UNSET;
	}

	/* new */
	idx = helper->strtab->len;
	g_byte_array_append(helper->strtab, (const guint8 *)str, str_len);
	g_hash_table_insert(helper->strtab_hash, g_strdup(str), GUINT_TO_POINTER(idx));
	return idx;
}

static gint
xb_builder_get_locale_priority(XbBuilderCompileHelper *helper, const gchar *locale)
{
	for (guint i = 0; i < helper->locales->len; i++) {
		const gchar *locale_tmp = g_ptr_array_index(helper->locales, i);
		if (g_strcmp0(locale_tmp, locale) == 0)
			return helper->locales->len - i;
	}
	return -1;
}

static void
xb_builder_compile_start_element_cb(GMarkupParseContext *context,
				    const gchar *element_name,
				    const gchar **attr_names,
				    const gchar **attr_values,
				    gpointer user_data,
				    GError **error)
{
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
	g_autoptr(XbBuilderNode) bn = xb_builder_node_new(element_name);

	/* check recursion depth to prevent stack exhaustion */
	if (helper->depth >= XB_BUILDER_MAX_DEPTH) {
		g_set_error(error,
			    G_IO_ERROR,
			    G_IO_ERROR_INVALID_DATA,
			    "nesting deeper than %u levels not supported",
			    (guint)XB_BUILDER_MAX_DEPTH);
		return;
	}
	helper->depth++;

	/* parent node is being ignored */
	if (helper->current != NULL &&
	    xb_builder_node_has_flag(helper->current, XB_BUILDER_NODE_FLAG_IGNORE))
		xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE);

	/* check if we should ignore the locale */
	if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE) &&
	    helper->compile_flags & XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS) {
		const gchar *xml_lang = NULL;
		for (guint i = 0; attr_names[i] != NULL; i++) {
			if (g_strcmp0(attr_names[i], "xml:lang") == 0) {
				xml_lang = attr_values[i];
				break;
			}
		}
		if (xml_lang == NULL) {
			if (helper->current != NULL) {
				gint prio = xb_builder_node_get_priority(helper->current);
				xb_builder_node_set_priority(bn, prio);
			}
		} else {
			gint prio = xb_builder_get_locale_priority(helper, xml_lang);
			if (prio < 0)
				xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE);
			xb_builder_node_set_priority(bn, prio);
		}
	}

	/* add attributes */
	if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) {
		for (guint i = 0; attr_names[i] != NULL; i++)
			xb_builder_node_set_attr(bn, attr_names[i], attr_values[i]);
	}

	/* add to tree */
	xb_builder_node_add_child(helper->current, bn);
	helper->current = bn;

	helper->elem_closed = FALSE;
}

static void
xb_builder_compile_end_element_cb(GMarkupParseContext *context,
				  const gchar *element_name,
				  gpointer user_data,
				  GError **error)
{
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
	g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(helper->current);
	if (parent == NULL) {
		g_set_error_literal(error,
				    G_IO_ERROR,
				    G_IO_ERROR_INVALID_DATA,
				    "Mismatched XML; no parent");
		return;
	}
	helper->current = parent;
	helper->depth--;
	helper->elem_closed = TRUE;
}

static void
xb_builder_compile_text_cb(GMarkupParseContext *context,
			   const gchar *text,
			   gsize text_len,
			   gpointer user_data,
			   GError **error)
{
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
	XbBuilderNode *bn = helper->current;
	XbBuilderNode *bc = xb_builder_node_get_last_child(bn);

	/* unimportant */
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return;

	/* repair text unless we know it's valid */
	if (helper->source_flags & XB_BUILDER_SOURCE_FLAG_LITERAL_TEXT)
		xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_LITERAL_TEXT);

	/* text or tail */
	if (!helper->elem_closed) {
		if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_HAS_TEXT)) {
			g_autoptr(GString) str = g_string_new(xb_builder_node_get_text(bn));
			g_string_append_len(str, text, text_len);
			xb_builder_node_set_text(bn, str->str, str->len);
		} else {
			xb_builder_node_set_text(bn, text, text_len);
		}
		return;
	}

	/* does this node have a child */
	if (bc != NULL) {
		xb_builder_node_set_tail(bc, text, text_len);
		return;
	}

	/* always set a tail, even if already set */
	xb_builder_node_set_tail(bn, text, text_len);
}

/**
 * xb_builder_import_source:
 * @self: a #XbSilo
 * @source: a #XbBuilderSource
 *
 * Adds a #XbBuilderSource to the #XbBuilder.
 *
 * Since: 0.1.0
 **/
void
xb_builder_import_source(XbBuilder *self, XbBuilderSource *source)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	g_autofree gchar *guid = NULL;

	g_return_if_fail(XB_IS_BUILDER(self));
	g_return_if_fail(XB_IS_BUILDER_SOURCE(source));

	/* get latest GUID */
	guid = xb_builder_source_get_guid(source);
	xb_builder_append_guid(self, guid);
	g_ptr_array_add(priv->sources, g_object_ref(source));
}

static gboolean
xb_builder_compile_source(XbBuilderCompileHelper *helper,
			  XbBuilderSource *source,
			  XbBuilderNode *root,
			  GCancellable *cancellable,
			  GError **error)
{
	GPtrArray *children;
	XbBuilderNode *info;
	gsize chunk_size = 32 * 1024;
	gssize len;
	g_autofree gchar *data = NULL;
	g_autofree gchar *guid = xb_builder_source_get_guid(source);
	g_autoptr(GPtrArray) children_copy = NULL;
	g_autoptr(GInputStream) istream = NULL;
	g_autoptr(GMarkupParseContext) ctx = NULL;
	g_autoptr(GTimer) timer = xb_silo_start_profile(helper->silo);
	g_autoptr(XbBuilderNode) root_tmp = xb_builder_node_new(NULL);
	const GMarkupParser parser = {xb_builder_compile_start_element_cb,
				      xb_builder_compile_end_element_cb,
				      xb_builder_compile_text_cb,
				      NULL,
				      NULL};

	/* add the source to a fake root in case it fails during processing */
	helper->current = root_tmp;
	helper->source_flags = xb_builder_source_get_flags(source);

	/* decompress */
	istream = xb_builder_source_get_istream(source, cancellable, error);
	if (istream == NULL)
		return FALSE;

	/* parse */
	ctx = g_markup_parse_context_new(&parser,
					 G_MARKUP_PREFIX_ERROR_POSITION |
					     G_MARKUP_TREAT_CDATA_AS_TEXT,
					 helper,
					 NULL);
	data = g_malloc(chunk_size);
	while ((len = g_input_stream_read(istream, data, chunk_size, cancellable, error)) > 0) {
		if (!g_markup_parse_context_parse(ctx, data, len, error))
			return FALSE;
	}
	if (len < 0)
		return FALSE;

	/* more opening than closing */
	if (root_tmp != helper->current) {
		g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "Mismatched XML");
		return FALSE;
	}

	/* run any node functions */
	if (!xb_builder_source_fixup(source, root_tmp, error))
		return FALSE;

	/* check to see if the root was ignored */
	if (xb_builder_node_has_flag(root_tmp, XB_BUILDER_NODE_FLAG_IGNORE)) {
		g_set_error_literal(error,
				    G_IO_ERROR,
				    G_IO_ERROR_INVALID_DATA,
				    "root node cannot be ignored");
		return FALSE;
	}

	/* a single root with no siblings was required */
	if (helper->compile_flags & XB_BUILDER_COMPILE_FLAG_SINGLE_ROOT) {
		if (xb_builder_node_get_children(root_tmp)->len > 1) {
			g_set_error_literal(error,
					    G_IO_ERROR,
					    G_IO_ERROR_INVALID_DATA,
					    "A root node without siblings was required");
			return FALSE;
		}
	}

	/* this is something we can query with later */
	info = xb_builder_source_get_info(source);
	if (info != NULL) {
		children = xb_builder_node_get_children(helper->current);
		for (guint i = 0; i < children->len; i++) {
			XbBuilderNode *bn = g_ptr_array_index(children, i);
			if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
				xb_builder_node_add_child(bn, info);
		}
	}

	/* add the children to the main document */
	children = xb_builder_node_get_children(root_tmp);
	children_copy = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
	for (guint i = 0; i < children->len; i++) {
		XbBuilderNode *bn = g_ptr_array_index(children, i);
		g_ptr_array_add(children_copy, g_object_ref(bn));
	}
	for (guint i = 0; i < children_copy->len; i++) {
		XbBuilderNode *bn = g_ptr_array_index(children_copy, i);
		xb_builder_node_unlink(bn);
		xb_builder_node_add_child(root, bn);
	}

	/* success */
	xb_silo_add_profile(helper->silo, timer, "compile %s", guid);
	return TRUE;
}

static gboolean
xb_builder_strtab_element_names_cb(XbBuilderNode *bn, gpointer user_data)
{
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
	const gchar *tmp;
	guint32 strtab_idx;

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return FALSE;
	tmp = xb_builder_node_get_element(bn);
	strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
	if (strtab_idx == XB_SILO_UNSET)
		return TRUE;
	xb_builder_node_set_element_idx(bn, strtab_idx);
	return FALSE;
}

static gboolean
xb_builder_strtab_attr_name_cb(XbBuilderNode *bn, gpointer user_data)
{
	GPtrArray *attrs;
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return FALSE;
	attrs = xb_builder_node_get_attrs(bn);
	for (guint i = 0; attrs != NULL && i < attrs->len; i++) {
		XbBuilderNodeAttr *attr = g_ptr_array_index(attrs, i);
		attr->name_idx = xb_builder_compile_add_to_strtab(helper, attr->name);
		if (attr->name_idx == XB_SILO_UNSET)
			return TRUE;
	}
	return FALSE;
}

static gboolean
xb_builder_strtab_attr_value_cb(XbBuilderNode *bn, gpointer user_data)
{
	GPtrArray *attrs;
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return FALSE;
	attrs = xb_builder_node_get_attrs(bn);
	for (guint i = 0; attrs != NULL && i < attrs->len; i++) {
		XbBuilderNodeAttr *attr = g_ptr_array_index(attrs, i);
		attr->value_idx = xb_builder_compile_add_to_strtab(helper, attr->value);
		if (attr->value_idx == XB_SILO_UNSET)
			return TRUE;
	}
	return FALSE;
}

static gboolean
xb_builder_strtab_text_cb(XbBuilderNode *bn, gpointer user_data)
{
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
	const gchar *tmp;
	guint32 strtab_idx;

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return FALSE;
	if (xb_builder_node_get_text(bn) != NULL) {
		tmp = xb_builder_node_get_text(bn);
		strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
		if (strtab_idx == XB_SILO_UNSET)
			return TRUE;
		xb_builder_node_set_text_idx(bn, strtab_idx);
	}
	if (xb_builder_node_get_tail(bn) != NULL) {
		tmp = xb_builder_node_get_tail(bn);
		strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
		if (strtab_idx == XB_SILO_UNSET)
			return TRUE;
		xb_builder_node_set_tail_idx(bn, strtab_idx);
	}
	return FALSE;
}

static gboolean
xb_builder_strtab_tokens_cb(XbBuilderNode *bn, gpointer user_data)
{
	XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
	GPtrArray *tokens = xb_builder_node_get_tokens(bn);
	guint32 strtab_idx;

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return FALSE;
	if (tokens == NULL)
		return FALSE;
	for (guint i = 0; i < MIN(tokens->len, XB_OPCODE_TOKEN_MAX); i++) {
		const gchar *tmp = g_ptr_array_index(tokens, i);
		if (tmp == NULL)
			continue;
		strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
		if (strtab_idx == XB_SILO_UNSET)
			return TRUE;
		xb_builder_node_add_token_idx(bn, strtab_idx);
	}
	return FALSE;
}

static gboolean
xb_builder_xml_lang_prio_cb(XbBuilderNode *bn, gpointer user_data)
{
	GPtrArray *nodes_to_destroy = (GPtrArray *)user_data;
	gint prio_best = 0;
	g_autoptr(GPtrArray) nodes = g_ptr_array_new();
	GPtrArray *siblings;
	g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(bn);

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;

	/* already ignored */
	if (xb_builder_node_get_priority(bn) == -2)
		return FALSE;

	/* get all the siblings with the same name */
	siblings = xb_builder_node_get_children(parent);
	for (guint i = 0; i < siblings->len; i++) {
		XbBuilderNode *bn2 = g_ptr_array_index(siblings, i);
		if (g_strcmp0(xb_builder_node_get_element(bn), xb_builder_node_get_element(bn2)) ==
		    0)
			g_ptr_array_add(nodes, bn2);
	}

	/* only one thing, so bail early */
	if (nodes->len == 1)
		return FALSE;

	/* find the best locale */
	for (guint i = 0; i < nodes->len; i++) {
		XbBuilderNode *bn2 = g_ptr_array_index(nodes, i);
		if (xb_builder_node_get_priority(bn2) > prio_best)
			prio_best = xb_builder_node_get_priority(bn2);
	}

	/* add any nodes not as good as the bext locale to the kill list */
	for (guint i = 0; i < nodes->len; i++) {
		XbBuilderNode *bn2 = g_ptr_array_index(nodes, i);
		if (xb_builder_node_get_priority(bn2) < prio_best)
			g_ptr_array_add(nodes_to_destroy, g_object_ref(bn2));

		/* never visit this node again */
		xb_builder_node_set_priority(bn2, -2);
	}

	return FALSE;
}

static gboolean
xb_builder_nodetab_size_cb(XbBuilderNode *bn, gpointer user_data)
{
	guint32 *sz = (guint32 *)user_data;

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return FALSE;
	*sz += xb_builder_node_size(bn) + 1; /* +1 for the sentinel */
	return FALSE;
}

typedef struct {
	GByteArray *buf;
} XbBuilderNodetabHelper;

static void
xb_builder_nodetab_write_sentinel(XbBuilderNodetabHelper *helper)
{
	XbSiloNode sn = {
	    .flags = XB_SILO_NODE_FLAG_NONE,
	    .attr_count = 0,
	};
	//	g_debug ("SENT @%u", (guint) helper->buf->len);
	g_byte_array_append(helper->buf, (const guint8 *)&sn, xb_silo_node_get_size(&sn));
}

#define XB_BUILDER_ATTR_MAX ((1 << 6) - 1)

static gboolean
xb_builder_nodetab_write_node(XbBuilderNodetabHelper *helper, XbBuilderNode *bn, GError **error)
{
	GPtrArray *attrs = xb_builder_node_get_attrs(bn);
	GArray *token_idxs = xb_builder_node_get_token_idxs(bn);
	XbSiloNode sn = {
	    .flags = XB_SILO_NODE_FLAG_IS_ELEMENT,
	    .attr_count = (attrs != NULL) ? attrs->len : 0,
	    .element_name = xb_builder_node_get_element_idx(bn),
	    .next = 0x0,
	    .parent = 0x0,
	    .text = xb_builder_node_get_text_idx(bn),
	    .tail = xb_builder_node_get_tail_idx(bn),
	    .token_count = 0,
	};

	/* sanity check */
	if (attrs != NULL && attrs->len > XB_BUILDER_ATTR_MAX) {
		g_set_error(error,
			    G_IO_ERROR,
			    G_IO_ERROR_NOT_SUPPORTED,
			    "too many attributes: %u",
			    attrs->len);
		return FALSE;
	}

	/* add tokens */
	if (token_idxs != NULL)
		sn.flags |= XB_SILO_NODE_FLAG_IS_TOKENIZED;

	/* if the node had no children and the text is just whitespace then
	 * remove it even in literal mode */
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_LITERAL_TEXT)) {
		if (xb_string_isspace(xb_builder_node_get_text(bn), -1))
			sn.text = XB_SILO_UNSET;
		if (xb_string_isspace(xb_builder_node_get_tail(bn), -1))
			sn.tail = XB_SILO_UNSET;
	}

	/* save this so we can set up the ->next pointers correctly */
	xb_builder_node_set_offset(bn, helper->buf->len);

	//	g_debug ("NODE @%u (%s)", (guint) helper->buf->len, xb_builder_node_get_element
	//(bn));

	/* there is no point adding more tokens than we can match */
	if (token_idxs != NULL)
		sn.token_count = MIN(token_idxs->len, XB_OPCODE_TOKEN_MAX);

	/* add to the buf */
	g_byte_array_append(helper->buf, (const guint8 *)&sn, sizeof(sn));

	/* add to the buf */
	for (guint i = 0; attrs != NULL && i < attrs->len; i++) {
		XbBuilderNodeAttr *ba = g_ptr_array_index(attrs, i);
		XbSiloNodeAttr attr = {
		    .attr_name = ba->name_idx,
		    .attr_value = ba->value_idx,
		};
		g_byte_array_append(helper->buf, (const guint8 *)&attr, sizeof(attr));
	}

	/* add tokens */
	for (guint i = 0; i < sn.token_count; i++) {
		guint32 idx = g_array_index(token_idxs, guint32, i);
		g_byte_array_append(helper->buf, (const guint8 *)&idx, sizeof(idx));
	}

	/* success */
	return TRUE;
}

static gboolean
xb_builder_nodetab_write(XbBuilderNodetabHelper *helper, XbBuilderNode *bn, GError **error)
{
	GPtrArray *children;

	/* ignore this */
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return TRUE;

	/* element */
	if (xb_builder_node_get_element(bn) != NULL) {
		if (!xb_builder_nodetab_write_node(helper, bn, error))
			return FALSE;
	}

	/* children */
	children = xb_builder_node_get_children(bn);
	for (guint i = 0; i < children->len; i++) {
		XbBuilderNode *bc = g_ptr_array_index(children, i);
		if (!xb_builder_nodetab_write(helper, bc, error))
			return FALSE;
	}

	/* sentinel */
	if (xb_builder_node_get_element(bn) != NULL)
		xb_builder_nodetab_write_sentinel(helper);

	/* success */
	return TRUE;
}

static XbSiloNode *
xb_builder_get_node(GByteArray *str, guint32 off)
{
	return (XbSiloNode *)(str->data + off);
}

static gboolean
xb_builder_nodetab_fix_cb(XbBuilderNode *bn, gpointer user_data)
{
	GPtrArray *siblings;
	XbBuilderNodetabHelper *helper = (XbBuilderNodetabHelper *)user_data;
	XbSiloNode *sn;
	gboolean found = FALSE;
	g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(bn);

	/* root node */
	if (xb_builder_node_get_element(bn) == NULL)
		return FALSE;
	if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
		return FALSE;

	/* get the position in the buffer */
	sn = xb_builder_get_node(helper->buf, xb_builder_node_get_offset(bn));
	if (sn == NULL)
		return FALSE;

	/* set the parent if the node has one */
	if (xb_builder_node_get_element(parent) != NULL)
		sn->parent = xb_builder_node_get_offset(parent);

	/* set ->next if the node has one */
	siblings = xb_builder_node_get_children(parent);
	for (guint i = 0; i < siblings->len; i++) {
		XbBuilderNode *bn2 = g_ptr_array_index(siblings, i);
		if (bn2 == bn) {
			found = TRUE;
			continue;
		}
		if (!found)
			continue;
		if (!xb_builder_node_has_flag(bn2, XB_BUILDER_NODE_FLAG_IGNORE)) {
			sn->next = xb_builder_node_get_offset(bn2);
			break;
		}
	}

	return FALSE;
}

static void
xb_builder_compile_helper_free(XbBuilderCompileHelper *helper)
{
	g_hash_table_unref(helper->strtab_hash);
	g_byte_array_unref(helper->strtab);
	g_clear_error(&helper->error);
	g_clear_object(&helper->silo);
	g_object_unref(helper->root);
	g_free(helper);
}

G_DEFINE_AUTOPTR_CLEANUP_FUNC(XbBuilderCompileHelper, xb_builder_compile_helper_free)

static gchar *
xb_builder_generate_guid(XbBuilder *self)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	XbGuid guid = {0x0};
	if (priv->guid->len > 0) {
		xb_guid_compute_for_data(&guid, (const guint8 *)priv->guid->str, priv->guid->len);
	}
	return xb_guid_to_string(&guid);
}

/**
 * xb_builder_import_node:
 * @self: a #XbSilo
 * @bn: a #XbBuilderNode
 *
 * Adds a node tree to the builder.
 *
 * If you are manually adding dynamic data sourced from a non-static source then you MUST use
 * xb_builder_append_guid() with the appropriate GUID value, e.g. the file name and mtime.
 *
 * If no appropriate value is available, the caller can use something like:
 *
 *     g_autofree gchar *tmp = xb_builder_node_export(bn, XB_NODE_EXPORT_FLAG_NONE, NULL);
 *     xb_builder_append_guid(builder, tmp);
 *
 * Failure to include an appropriate GUID value would allow an out-of-data silo to be used.
 *
 * Since: 0.1.0
 **/
void
xb_builder_import_node(XbBuilder *self, XbBuilderNode *bn)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	g_return_if_fail(XB_IS_BUILDER(self));
	g_return_if_fail(XB_IS_BUILDER_NODE(bn));
	g_ptr_array_add(priv->nodes, g_object_ref(bn));
}

/**
 * xb_builder_add_locale:
 * @self: a #XbSilo
 * @locale: a locale, e.g. "en_US"
 *
 * Adds a locale to the builder. Locales added first will be prioritised over
 * locales added later.
 *
 * Since: 0.1.0
 **/
void
xb_builder_add_locale(XbBuilder *self, const gchar *locale)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);

	g_return_if_fail(XB_IS_BUILDER(self));
	g_return_if_fail(locale != NULL);

	if (g_str_has_suffix(locale, ".UTF-8"))
		return;
	for (guint i = 0; i < priv->locales->len; i++) {
		const gchar *locale_tmp = g_ptr_array_index(priv->locales, i);
		if (g_strcmp0(locale_tmp, locale) == 0)
			return;
	}
	g_ptr_array_add(priv->locales, g_strdup(locale));

	/* if the user changes LANG, the blob is no longer valid */
	xb_builder_append_guid(self, locale);
}

static gboolean
xb_builder_watch_source(XbBuilder *self,
			XbBuilderSource *source,
			XbSilo *silo,
			GCancellable *cancellable,
			GError **error)
{
	GFile *file = xb_builder_source_get_file(source);
	g_autoptr(GFile) watched_file = NULL;
	if (file == NULL)
		return TRUE;
	if ((xb_builder_source_get_flags(source) &
	     (XB_BUILDER_SOURCE_FLAG_WATCH_FILE | XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY)) == 0)
		return TRUE;

	if (xb_builder_source_get_flags(source) & XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY)
		watched_file = g_file_get_parent(file);
	else
		watched_file = g_object_ref(file);

	return xb_silo_watch_file(silo, watched_file, cancellable, error);
}

static gboolean
xb_builder_watch_sources(XbBuilder *self, XbSilo *silo, GCancellable *cancellable, GError **error)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	for (guint i = 0; i < priv->sources->len; i++) {
		XbBuilderSource *source = g_ptr_array_index(priv->sources, i);
		if (!xb_builder_watch_source(self, source, silo, cancellable, error))
			return FALSE;
	}
	return TRUE;
}

/**
 * xb_builder_compile:
 * @self: a #XbSilo
 * @flags: some #XbBuilderCompileFlags, e.g. %XB_BUILDER_SOURCE_FLAG_LITERAL_TEXT
 * @cancellable: a #GCancellable, or %NULL
 * @error: the #GError, or %NULL
 *
 * Compiles a #XbSilo.
 *
 * Returns: (transfer full): a #XbSilo, or %NULL for error
 *
 * Since: 0.1.0
 **/
XbSilo *
xb_builder_compile(XbBuilder *self,
		   XbBuilderCompileFlags flags,
		   GCancellable *cancellable,
		   GError **error)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	guint32 nodetabsz = sizeof(XbSiloHeader);
	g_autoptr(GByteArray) buf = NULL;
	g_autoptr(GBytes) blob = NULL;
	XbSiloHeader *hdrptr;
	XbSiloHeader hdr = {
	    .magic = XB_SILO_MAGIC_BYTES,
	    .version = XB_SILO_VERSION,
	    .strtab = 0,
	    .strtab_ntags = 0,
	    .padding = {0x0},
	    .guid = {0x0},
	    .filesz = 0x0,
	};
	XbBuilderNodetabHelper nodetab_helper = {
	    .buf = NULL,
	};
	g_autoptr(GPtrArray) nodes_to_destroy =
	    g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
	g_autoptr(GTimer) timer = NULL;
	g_autoptr(XbBuilderCompileHelper) helper = NULL;

	g_return_val_if_fail(XB_IS_BUILDER(self), NULL);
	g_return_val_if_fail(cancellable == NULL || G_IS_CANCELLABLE(cancellable), NULL);
	g_return_val_if_fail(error == NULL || *error == NULL, NULL);

	/* this is inferred */
	if (flags & XB_BUILDER_COMPILE_FLAG_SINGLE_LANG)
		flags |= XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS;

	/* the builder needs to know the locales */
	if (priv->locales->len == 0 && (flags & XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS)) {
		const gchar *const *locales = g_get_language_names();
		for (guint i = 0; locales[i] != NULL; i++)
			xb_builder_add_locale(self, locales[i]);
	}

	/* create helper used for compiling */
	helper = g_new0(XbBuilderCompileHelper, 1);
	helper->compile_flags = flags;
	helper->root = xb_builder_node_new(NULL);
	helper->silo = xb_silo_new();
	helper->locales = priv->locales;
	helper->strtab = g_byte_array_new();
	helper->strtab_hash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
	helper->elem_closed = FALSE;

	/* for profiling */
	xb_silo_set_profile_flags(helper->silo, priv->profile_flags);
	timer = xb_silo_start_profile(helper->silo);

	/* build node tree */
	for (guint i = 0; i < priv->sources->len; i++) {
		XbBuilderSource *source = g_ptr_array_index(priv->sources, i);
		const gchar *prefix = xb_builder_source_get_prefix(source);
		g_autofree gchar *source_guid = xb_builder_source_get_guid(source);
		g_autoptr(XbBuilderNode) root = NULL;
		g_autoptr(GError) error_local = NULL;

		/* find, or create the prefix */
		if (prefix != NULL) {
			root = xb_builder_node_get_child(helper->root, prefix, NULL);
			if (root == NULL)
				root = xb_builder_node_insert(helper->root, prefix, NULL);
		} else {
			/* don't allow damaged XML files to ruin all the next ones */
			root = g_object_ref(helper->root);
		}

		/* watch the source */
		if (!xb_builder_watch_source(self, source, helper->silo, cancellable, error))
			return NULL;

		if (priv->profile_flags & XB_SILO_PROFILE_FLAG_DEBUG)
			g_debug("compiling %s…", source_guid);
		if (!xb_builder_compile_source(helper, source, root, cancellable, &error_local)) {
			if (flags & XB_BUILDER_COMPILE_FLAG_IGNORE_INVALID) {
				g_debug("ignoring invalid file %s: %s",
					source_guid,
					error_local->message);
				continue;
			}
			g_propagate_prefixed_error(error,
						   g_steal_pointer(&error_local),
						   "failed to compile %s: ",
						   source_guid);
			return NULL;
		}
	}

	/* run any node functions */
	for (guint i = 0; i < priv->fixups->len; i++) {
		XbBuilderFixup *fixup = g_ptr_array_index(priv->fixups, i);
		if (!xb_builder_fixup_node(fixup, helper->root, error))
			return NULL;
	}

	/* only include the highest priority translation */
	if (flags & XB_BUILDER_COMPILE_FLAG_SINGLE_LANG) {
		xb_builder_node_traverse(helper->root,
					 G_PRE_ORDER,
					 G_TRAVERSE_ALL,
					 -1,
					 xb_builder_xml_lang_prio_cb,
					 nodes_to_destroy);
		for (guint i = 0; i < nodes_to_destroy->len; i++) {
			XbBuilderNode *bn = g_ptr_array_index(nodes_to_destroy, i);
			xb_builder_node_unlink(bn);
		}
		xb_silo_add_profile(helper->silo, timer, "filter single-lang");
	}

	/* add any manually build nodes */
	for (guint i = 0; i < priv->nodes->len; i++) {
		XbBuilderNode *bn = g_ptr_array_index(priv->nodes, i);
		xb_builder_node_add_child(helper->root, bn);
	}

	/* get the size of the nodetab */
	xb_builder_node_traverse(helper->root,
				 G_PRE_ORDER,
				 G_TRAVERSE_ALL,
				 -1,
				 xb_builder_nodetab_size_cb,
				 &nodetabsz);
	buf = g_byte_array_sized_new(nodetabsz);
	xb_silo_add_profile(helper->silo, timer, "get size nodetab");

	/* add everything to the strtab */
	xb_builder_node_traverse(helper->root,
				 G_PRE_ORDER,
				 G_TRAVERSE_ALL,
				 -1,
				 xb_builder_strtab_element_names_cb,
				 helper);
	if (helper->error != NULL) {
		g_propagate_error(error, g_steal_pointer(&helper->error));
		return NULL;
	}
	if (g_hash_table_size(helper->strtab_hash) > G_MAXUINT16) {
		g_set_error_literal(error,
				    G_IO_ERROR,
				    G_IO_ERROR_INVALID_DATA,
				    "too many unique element names for strtab");
		return NULL;
	}
	hdr.strtab_ntags = (guint16)g_hash_table_size(helper->strtab_hash);
	xb_silo_add_profile(helper->silo, timer, "adding strtab element");
	xb_builder_node_traverse(helper->root,
				 G_PRE_ORDER,
				 G_TRAVERSE_ALL,
				 -1,
				 xb_builder_strtab_attr_name_cb,
				 helper);
	if (helper->error != NULL) {
		g_propagate_error(error, g_steal_pointer(&helper->error));
		return NULL;
	}
	xb_silo_add_profile(helper->silo, timer, "adding strtab attr name");
	xb_builder_node_traverse(helper->root,
				 G_PRE_ORDER,
				 G_TRAVERSE_ALL,
				 -1,
				 xb_builder_strtab_attr_value_cb,
				 helper);
	if (helper->error != NULL) {
		g_propagate_error(error, g_steal_pointer(&helper->error));
		return NULL;
	}
	xb_silo_add_profile(helper->silo, timer, "adding strtab attr value");
	xb_builder_node_traverse(helper->root,
				 G_PRE_ORDER,
				 G_TRAVERSE_ALL,
				 -1,
				 xb_builder_strtab_text_cb,
				 helper);
	if (helper->error != NULL) {
		g_propagate_error(error, g_steal_pointer(&helper->error));
		return NULL;
	}
	xb_silo_add_profile(helper->silo, timer, "adding strtab text");
	xb_builder_node_traverse(helper->root,
				 G_PRE_ORDER,
				 G_TRAVERSE_ALL,
				 -1,
				 xb_builder_strtab_tokens_cb,
				 helper);
	if (helper->error != NULL) {
		g_propagate_error(error, g_steal_pointer(&helper->error));
		return NULL;
	}
	xb_silo_add_profile(helper->silo, timer, "adding strtab tokens");

	/* add the initial header */
	hdr.strtab = nodetabsz;
	if (priv->guid->len > 0) {
		XbGuid guid_tmp;
		xb_guid_compute_for_data(&guid_tmp,
					 (const guint8 *)priv->guid->str,
					 priv->guid->len);
		memcpy(&hdr.guid, &guid_tmp, sizeof(guid_tmp));
	}
	g_byte_array_append(buf, (const guint8 *)&hdr, sizeof(hdr));

	/* write nodes to the nodetab */
	nodetab_helper.buf = buf;
	if (!xb_builder_nodetab_write(&nodetab_helper, helper->root, error))
		return NULL;
	xb_silo_add_profile(helper->silo, timer, "writing nodetab");

	/* set all the ->next and ->parent offsets */
	xb_builder_node_traverse(helper->root,
				 G_PRE_ORDER,
				 G_TRAVERSE_ALL,
				 -1,
				 xb_builder_nodetab_fix_cb,
				 &nodetab_helper);
	xb_silo_add_profile(helper->silo, timer, "fixing ->parent and ->next");

	/* append the string table */
	g_byte_array_append(buf, (const guint8 *)helper->strtab->data, helper->strtab->len);
	xb_silo_add_profile(helper->silo, timer, "appending strtab");

	/* update the file size */
	hdrptr = (XbSiloHeader *)buf->data;
	hdrptr->filesz = buf->len;

	/* create data */
	blob = g_byte_array_free_to_bytes(g_steal_pointer(&buf));
	if (!xb_silo_load_from_bytes(helper->silo, blob, XB_SILO_LOAD_FLAG_NONE, error))
		return NULL;

	/* success */
	return g_steal_pointer(&helper->silo);
}

/**
 * xb_builder_ensure:
 * @self: a #XbSilo
 * @file: a #GFile
 * @flags: some #XbBuilderCompileFlags, e.g. %XB_BUILDER_COMPILE_FLAG_IGNORE_INVALID
 * @cancellable: a #GCancellable, or %NULL
 * @error: the #GError, or %NULL
 *
 * Ensures @file is up to date, and returns a compiled #XbSilo.
 *
 * If @silo is being used by a query (e.g. in another thread) then all node
 * data is immediately invalid.
 *
 * The returned #XbSilo will use the thread-default main context at the time of
 * calling this function for its future signal emissions.
 *
 * Returns: (transfer full): a #XbSilo, or %NULL for error
 *
 * Since: 0.1.0
 **/
XbSilo *
xb_builder_ensure(XbBuilder *self,
		  GFile *file,
		  XbBuilderCompileFlags flags,
		  GCancellable *cancellable,
		  GError **error)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	XbSiloLoadFlags load_flags = XB_SILO_LOAD_FLAG_NONE;
	g_autofree gchar *fn = NULL;
	g_autoptr(XbSilo) silo_tmp = xb_silo_new();
	g_autoptr(XbSilo) silo = NULL;
	g_autoptr(GError) error_local = NULL;

	g_return_val_if_fail(XB_IS_BUILDER(self), NULL);
	g_return_val_if_fail(G_IS_FILE(file), NULL);
	g_return_val_if_fail(cancellable == NULL || G_IS_CANCELLABLE(cancellable), NULL);
	g_return_val_if_fail(error == NULL || *error == NULL, NULL);

	/* watch the blob, so propagate flags */
	if (flags & XB_BUILDER_COMPILE_FLAG_WATCH_BLOB) {
		load_flags |= XB_SILO_LOAD_FLAG_WATCH_BLOB;
		if (!xb_silo_watch_file(silo_tmp, file, cancellable, error))
			return NULL;
	}

	/* ensure all the sources are watched */
	if (!xb_builder_watch_sources(self, silo_tmp, cancellable, error))
		return NULL;

	/* profile new silo if needed */
	xb_silo_set_profile_flags(silo_tmp, priv->profile_flags);

	/* load the file and peek at the GUIDs */
	fn = g_file_get_path(file);
	g_debug("attempting to load %s", fn);
	if (!xb_silo_load_from_file(silo_tmp,
				    file,
				    XB_SILO_LOAD_FLAG_NONE,
				    cancellable,
				    &error_local)) {
		g_debug("failed to load silo: %s", error_local->message);
	} else {
		g_autofree gchar *guid = xb_builder_generate_guid(self);
		if (priv->profile_flags & XB_SILO_PROFILE_FLAG_DEBUG)
			g_debug("GUID string: %s", priv->guid->str);
		g_debug("file: %s, current:%s", xb_silo_get_guid(silo_tmp), guid);

		/* no compile required */
		if (g_strcmp0(xb_silo_get_guid(silo_tmp), guid) == 0 ||
		    (flags & XB_BUILDER_COMPILE_FLAG_IGNORE_GUID) > 0) {
			g_debug("loading silo with existing file contents");
			return g_steal_pointer(&silo_tmp);
		}
	}

	/* fallback to just creating a new file */
	silo = xb_builder_compile(self, flags, cancellable, error);
	if (silo == NULL)
		return NULL;

	/* this might seem unnecessary, but windows cannot do _chsize() (introduced in GLib commit
	 * https://gitlab.gnome.org/GNOME/glib/-/commit/3f705ffa1230757b910a06a705104d4b0fee2c05)
	 * on a mmap'd file -- so manually tear that down before writing the new file */
	g_clear_object(&silo_tmp);
	if (!xb_silo_save_to_file(silo, file, NULL, error))
		return NULL;

	/* load from a file to re-mmap it */
	if (!xb_silo_load_from_file(silo, file, load_flags, cancellable, error))
		return NULL;

	/* ensure all the sources are watched on the reloaded silo */
	if (!xb_builder_watch_sources(self, silo, cancellable, error))
		return NULL;

	/* success */
	return g_steal_pointer(&silo);
}

/**
 * xb_builder_append_guid:
 * @self: a #XbSilo
 * @guid: any text, typcically a filename or GUID
 *
 * Adds the GUID to the internal correctness hash.
 *
 * Since: 0.1.0
 **/
void
xb_builder_append_guid(XbBuilder *self, const gchar *guid)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	if (priv->guid->len > 0)
		g_string_append(priv->guid, "&");
	g_string_append(priv->guid, guid);
}

/**
 * xb_builder_set_profile_flags:
 * @self: a #XbBuilder
 * @profile_flags: some #XbSiloProfileFlags, e.g. %XB_SILO_PROFILE_FLAG_DEBUG
 *
 * Enables or disables the collection of profiling data.
 *
 * Since: 0.1.1
 **/
void
xb_builder_set_profile_flags(XbBuilder *self, XbSiloProfileFlags profile_flags)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	g_return_if_fail(XB_IS_BUILDER(self));
	priv->profile_flags = profile_flags;
}

/**
 * xb_builder_add_fixup:
 * @self: a #XbBuilder
 * @fixup: a #XbBuilderFixup
 *
 * Adds a function that will get run on every #XbBuilderNode compile creates
 * for the silo. This is run after all the #XbBuilderSource fixups have been
 * run.
 *
 * Since: 0.1.3
 **/
void
xb_builder_add_fixup(XbBuilder *self, XbBuilderFixup *fixup)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	g_autofree gchar *guid = NULL;

	g_return_if_fail(XB_IS_BUILDER(self));
	g_return_if_fail(XB_IS_BUILDER_FIXUP(fixup));

	/* append function IDs */
	guid = xb_builder_fixup_get_guid(fixup);
	xb_builder_append_guid(self, guid);
	g_ptr_array_add(priv->fixups, g_object_ref(fixup));
}

static void
xb_builder_finalize(GObject *obj)
{
	XbBuilder *self = XB_BUILDER(obj);
	XbBuilderPrivate *priv = GET_PRIVATE(self);

	g_ptr_array_unref(priv->sources);
	g_ptr_array_unref(priv->nodes);
	g_ptr_array_unref(priv->locales);
	g_ptr_array_unref(priv->fixups);
	g_string_free(priv->guid, TRUE);

	G_OBJECT_CLASS(xb_builder_parent_class)->finalize(obj);
}

static void
xb_builder_class_init(XbBuilderClass *klass)
{
	GObjectClass *object_class = G_OBJECT_CLASS(klass);
	object_class->finalize = xb_builder_finalize;
}

static void
xb_builder_init(XbBuilder *self)
{
	XbBuilderPrivate *priv = GET_PRIVATE(self);
	priv->sources = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
	priv->nodes = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
	priv->fixups = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
	priv->locales = g_ptr_array_new_with_free_func(g_free);
	priv->guid = g_string_new(xb_version_string());
}

/**
 * xb_builder_new:
 *
 * Creates a new builder.
 *
 * The #XbSilo returned by the methods of this #XbBuilder will use the
 * thread-default main context at the time of calling this function for its
 * future signal emissions.
 *
 * Returns: a new #XbBuilder
 *
 * Since: 0.1.0
 **/
XbBuilder *
xb_builder_new(void)
{
	return g_object_new(XB_TYPE_BUILDER, NULL);
}
