OurBigBook Docs
This is a central source file that basically contains all the functionality of the OurBigBook Library, so basically the OurBigBook Markup-to-whatever (e.g. HTML) conversion code, including parsing and rendering.
Things that are not there are things that only use markup conversion, e.g.:
- OurBigBook CLI: does conversion from command line
- OurBigBook Web
This file must be able to run in the browser, so it must not contain any Node.js specifics.
It exposes the central
convert function for markup conversion.You should normally use the packaged
_obb/ourbigbook.js version of this file when using ourbigbook as an external dependency.This file is large, and large text files are not previewed, as they would take up too much useless vertical space and disk memory/bandwidth.
index.js
let dolog = false
const globals = {};
if (typeof performance === 'undefined') {
// Fuck, I can't find how to make this browser/node portable more nicely.
// https://github.com/nodejs/node/issues/28635
// https://github.com/browserify/perf-hooks-browserify
//
// eval('require') because react-scripts build from web/
// calls webpack, which for some reason cannot find it.
globals.performance = eval('require')('perf_hooks').performance;
} else {
globals.performance = performance;
}
const katex = require('katex');
// This tells webpack to pull in the KaTeX mhchem extension
// https://github.com/KaTeX/KaTeX/tree/main/contrib/mhchem
require('katex/contrib/mhchem');
const lodash = require('lodash');
const path = require('path');
const pluralize = require('pluralize');
const runtime_common = require('./runtime_common');
const { kMaxLength } = require('buffer');
// consts used by classes.
const HTML_PARENT_MARKER = '<span class="fa-solid-900 icon">\u{f062}</span>';
exports.HTML_PARENT_MARKER = HTML_PARENT_MARKER;
const NOSPLIT_MARKER_TEXT = 'nosplit'
exports.NOSPLIT_MARKER_TEXT = NOSPLIT_MARKER_TEXT;
const SPLIT_MARKER_TEXT = 'split'
exports.SPLIT_MARKER_TEXT = SPLIT_MARKER_TEXT;
class AstNode {
/**
* Abstract syntax tree node. This is the base node type that
* represents the parsed output.
*
* @param {AstType} node_type -
* @param {String} macro_name - - if node_type === AstType.PLAINTEXT or AstType.ERROR: fixed to
* AstType.PLAINTEXT_MACRO_NAME
* - elif node_type === AstType.PARAGRAPH: fixed to undefined
* - else: arbitrary regular macro
* @param {Object[String, AstArgument]} args - dict of arg names to arguments.
* where arguments are arrays of AstNode
* @param {SourceLocation} source_location - the best representation of where the macro is starts in the document
* used primarily to present useful debug messages
* @param {Object} options
* {String} text - the text content of an AstType.PLAINTEXT, undefined for other types
*/
constructor(node_type, macro_name, args, source_location, options={}) {
options = { ...options }
if (!('count_words' in options)) {
options.count_words = true;
}
if (!('first_toplevel_child' in options)) {
options.first_toplevel_child = false;
}
if (!('from_include' in options)) {
options.from_include = false;
}
if (!('from_ourbigbook_example' in options)) {
options.from_ourbigbook_example = false;
}
if (!('force_no_index' in options)) {
options.force_no_index = false;
}
if (!(Macro.ID_ARGUMENT_NAME in options)) {
options.id = undefined;
}
if (!('header_parent_ids' in options)) {
options.header_parent_ids = [];
}
if (!('header_tree_node_word_count' in options)) {
options.header_tree_node_word_count = 0;
}
if (!('is_first_header_in_input_file' in options)) {
// Either the first header on a regular toplevel input,
// or the first header inside an included include.
options.is_first_header_in_input_file = false;
}
if (!('numbered' in options)) {
options.numbered = true;
}
if (!('parent_ast' in options)) {
// AstNode. This is different from header_tree_node because
// it points to the parent ast node, i.e. the ast_node that
// contains this inside one of its arguments.
//
// header_tree_node on the other hand points to the header tree.
// The header tree is currently not even connected via arguments.
options.parent_ast = undefined;
}
if (!('parent_ast_index' in options)) {
options.parent_ast_index = undefined
}
if (!('split_default' in options)) {
options.split_default = false;
}
if (!('split_default_children' in options)) {
// This header is not split_default, but its children are.
options.split_default_children = false;
}
if (!('synonym' in options)) {
options.synonym = undefined;
}
if (!('text' in options)) {
options.text = undefined;
}
if (!('xss_safe' in options)) {
// If true, force this particular instance of the macro to be safe.
// even if the macro is unsafe by default.
// This is used for autogenerated elements like incoming links, which.
// are guaranteed to be safe through other means.
options.xss_safe = false;
}
// Generic fields.
this.node_type = node_type;
this.macro_name = macro_name;
// For elements that have an id.
// {String} or undefined.
// Includes scope. Ideally, we should remove that requirement to not duplicate.
// the information. But it will require a large hard refactor... lazy.
this.id = options.id;
// Current running scope. This is inherited from the scope of the ancestor headers.
// An element with {scope} set does not get this option set (except if due to ancestors),
// only its children do.
this.scope = options.scope;
this.subdir = options.subdir;
// For elements that are of AstType.PLAINTEXT.
this.text = options.text
this.source_location = source_location;
this.args = args;
// The effective path of the the file counting from the toplevel directory,
// coming either from the file=XXX or title if that is empty.
// Examples:
// - _file/path/to/myfile.txt.bigb: .file = path/to/myfile.txt, without the _file prefix
this.file = undefined
this.first_toplevel_child = options.first_toplevel_child;
this.is_first_header_in_input_file = options.is_first_header_in_input_file;
// This is the Nth macro of this type that appears in the document.
this.macro_count = undefined;
// A unique global index for this macro.
this.macro_count_global = undefined;
// This is the Nth macro of this type that is visible,
// and therefore increments counts such as Figure 1), Figure 2), etc.
// All indexed IDs (those that can be linked to) are also visible, but
// it is possible to force non-indexed IDs to count as well with
// captionNumberVisible.
this.macro_count_visible = undefined;
// Is this header numbered? Note that this is not set based on the numbered= argument:
// that argument determines this .numbered of descendants only. This is analogous to scope.
this.numbered = options.numbered;
// {AstNode} that contains this as an argument.
this.parent_ast = options.parent_ast;
// AstArgument
this.parent_argument = undefined;
this.split_default = options.split_default;
this.split_default_children = options.split_default_children;
// {HeaderTreeNode} that points to the element.
// This is used for both headers and non headers:
// the only difference is that non-headers are not connected as
// children of their parent. But they still know who the parent is.
// This was originally required for header scope resolution.
this.header_tree_node = options.header_tree_node;
// For DB serialization since we don't current serialize header_tree_node.
this.header_tree_node_word_count = options.header_tree_node_word_count
// When fetching Nodes from the database, we only get their ID,
// so we can't construct a full proper header_tree_node from that alone.
// So we just store the IDs here and not on header_tree_node as that
// alone is already useful.
this.header_parent_ids = options.header_parent_ids;
this.validation_error = undefined;
this.validation_output = {};
// The ID of this element has been indexed.
this.index_id = undefined;
this.force_no_index = options.force_no_index;
// The ToC will go before this header.
this.toc_header = false;
// This ast is a descendant of a header. Applications:
// - header descendants don't count for word counts
this.in_header = false;
// {String} the id of the target of this synonym header.
// undefined if not a synonym.
this.synonym = options.synonym;
// This was added to the tree from an include.
this.from_include = options.from_include;
// This was added to the tree from a ourbigbook_example.
this.from_ourbigbook_example = options.from_ourbigbook_example;
// Includes under this header.
this.includes = [];
this.xss_safe = options.xss_safe;
// Array of AstNode of synonym headers of this one that have title2 set.
this.title2s = []
this.toplevel_id = undefined
// Has this node already been validated? This cannot happen twice e.g. because of booleans:
// booleans currently just work by adding a dummy argument with a default. But then we lose
// the fact if the thing was given or not. This matters for example for numbered, which
// inherits from parent if the arg is not given on parent.
this.validated = false
// string. If set, the ast will render as this string and nothing else.
// This was originally introduced to prevent invalid HTML output e.g.
// in errors such as \a inside \a. We never want invalid HTML even when
// such errors are detected.
this.renderAsError = undefined
// Should automatic topic links be applied to this ast.
// Only makes sense for plaintext nodes.
this.automaticTopicLinks = false
this.count_words = options.count_words
if (this.node_type === AstType.PLAINTEXT) {
this.word_count = this.text.split(/\s+/).filter(i => i).length;
} else {
this.word_count = 0;
}
for (const argname in args) {
const arg = args[argname]
this.setup_argument(argname, arg);
for (const ast of arg) {
ast.parent_ast = this
}
}
}
/**
* Convert this AST node to an output string.
*
* @param {Object} context
* If a call will change this object, it must first make a copy,
* otherwise future calls to non descendants would also be affected by the change.
*
* - {Object} options - global options passed in from toplevel. Never modified by calls
* - {bool} html_is_attr - are we inside an HTML attribute, which implies implies different
* escape rules, e.g. " and ' must be escaped.
* - {bool} html_escape - if false, disable HTML escaping entirely. This is needed for
* content that is passed for an external tool for processing, for example
* Math equations to KaTeX, In that case, the arguments need to be passed as is,
* otherwise e.g. `1 < 2` would escape the `<` to `<` and KaTeX would receive bad input.
* - {HeaderTreeNode} header_tree - HeaderTreeNode graph containing AstNode headers
* - {Object} ids - map of document IDs to their description:
* - 'prefix': prefix to add for a full reference, e.g. `Figure 1`, `Section 2`, etc.
* - {AstArgument} 'title': the title of the element linked to
* - {bool} in_caption_number_visible
* - {String} root_relpath_shift - relative path introduced due to a scope in split header mode
* @param {Object} context
* @return {String}
*/
render(context={}) {
if (this.renderAsError) {
return errorMessageInOutput(this.renderAsError)
}
if (!('macros' in context)) {
throw new Error('context does not have a mandatory .macros property');
}
if (!('source_location' in context)) {
// Set a source location to this element and the entire subtree.
// This allows for less verbose manual construction of trees with
// a single dummy source_location.
context.source_location = undefined;
}
if (!('validateAst' in context)) {
// Do validateAst to this element and the entire subtree.
// This allows for less verbose manual construction of trees with
// a single dummy source_location.
context.validateAst = false;
}
if (
(
this.from_include &&
context.in_split_headers &&
!this.from_ourbigbook_example
) ||
(
context.options.output_format === OUTPUT_FORMAT_OURBIGBOOK &&
this.from_ourbigbook_example &&
!context.id_conversion
)
) {
return '';
}
if (context.source_location !== undefined) {
this.source_location = context.source_location;
for (const argname in this.args) {
this.args[argname].source_location = context.source_location;
}
}
if (context.validateAst && !this.validated) {
validateAst(this, context);
}
if (this.node_type === AstType.PARAGRAPH || this.node_type === AstType.NEWLINE) {
// Possible for AstType === PARAGRAPH which can happen for
// shorthand paragraph inside header or ID during post processing.
// Not the nicest solution, but prevents the crash, so so be it.
// https://github.com/ourbigbook/ourbigbook/issues/143
return ' '
}
const macro = context.macros[this.macro_name];
let out, render_pre, render_post
if (this.validation_error === undefined) {
let output_format;
if (context.id_conversion) {
output_format = OUTPUT_FORMAT_ID;
} else {
output_format = context.options.output_format;
}
let convert_function
if (!macro.options.xss_safe && (this.xss_safe || context.options.xss_safe)) {
let xss_safe_alt = macro.options.xss_safe_alt
if (xss_safe_alt === undefined) {
xss_safe_alt = XSS_SAFE_ALT_DEFAULT
}
convert_function = xss_safe_alt[output_format]
} else {
convert_function = macro.convert_funcs[output_format];
}
if (convert_function === undefined) {
const message = `output format ${context.options.output_format} not defined for macro ${this.macro_name}`;
renderError(context, message, this.source_location);
out = errorMessageInOutput(message, context);
} else {
const opts = {
extra_returns: {}
}
out = convert_function(this, context, opts);
render_pre = opts.extra_returns.render_pre
render_post = opts.extra_returns.render_post
}
} else {
renderError(
context,
this.validation_error[0],
this.validation_error[1],
this.validation_error[2]
);
out = errorMessageInOutput(this.validation_error[0], context);
}
// Add a div to all direct children of toplevel to implement
// the on hover links to self and left margin.
{
const parent_ast = this.parent_ast;
if (
parent_ast !== undefined &&
parent_ast.macro_name === Macro.TOPLEVEL_MACRO_NAME &&
this.id !== undefined &&
macro.toplevel_link
) {
out = OUTPUT_FORMATS[context.options.output_format].toplevelChildModifier(this, context, out);
if (render_pre) {
out = render_pre + out
}
if (render_post) {
out = out + render_post
}
}
}
context.last_render = out
if (
context.toplevel_output_path &&
this.macro_name === Macro.HEADER_MACRO_NAME &&
this.is_first_header_in_input_file &&
context.extra_returns.rendered_outputs[context.toplevel_output_path] !== undefined &&
context.extra_returns.rendered_outputs[context.toplevel_output_path].h1RenderLength === undefined
) {
// TODO This is a bit of a hack used for web where we want separate headers.
// Ideally we should just render h1 and body separately neatly before merging the
// two sources, e.g. at renderAstList. But lazy. Let's build on top of some technical debt.
context.extra_returns.rendered_outputs[context.toplevel_output_path].h1RenderLength = out.length
}
return out;
}
add_argument(argname, arg) {
this.args[argname] = arg
this.setup_argument(argname, arg)
}
/** Get all ancestors Asts of this Ast ordered from nearest to furthest.
* @return {List[AstNode]}.
*/
ancestors(context) {
const ancestors = [];
let cur_ast = this
let ancestor_id_set = new Set()
while (true) {
cur_ast = cur_ast.get_header_parent_asts(context)[0];
if (cur_ast === undefined) {
break
}
if (ancestor_id_set.has(cur_ast.id)) {
// TODO currently only exercised on web, e.g. when setting a parentId on index article.
// In that case this is because we render in "convert" before we know if it is index
// or not, because we determine if it is index or not in the h1Only conversion inside convert.
// This should be refactored a bit and then we can remove this, but lazy now.
break
} else {
ancestor_id_set.add(cur_ast.id)
}
ancestors.push(cur_ast);
}
return ancestors
}
// Return the full scope of a given node. This includes the concatenation of both:
// * any scopes of any parents
// * the ID of the node if it has a scope set for itself
// If none of those provide scopes, return undefined.
calculate_scope() {
let parent_scope;
if (this.scope !== undefined) {
parent_scope = this.scope;
}
if (this.subdir) {
if (parent_scope) {
parent_scope += Macro.HEADER_SCOPE_SEPARATOR
} else {
parent_scope = ''
}
parent_scope += this.subdir
}
let self_scope;
if (
this.validation_output.scope !== undefined &&
this.validation_output.scope.boolean
) {
self_scope = this.id;
if (parent_scope !== undefined) {
self_scope = self_scope.substring(parent_scope.length + 1)
}
} else {
self_scope = '';
}
let ret = '';
if (parent_scope !== undefined) {
ret += parent_scope;
}
if (
parent_scope !== undefined &&
self_scope !== ''
) {
ret += Macro.HEADER_SCOPE_SEPARATOR;
}
if (self_scope !== '') {
ret += self_scope;
}
if (ret === '') {
return undefined;
}
return ret;
}
/* Get parent ID, but only consider IDs that come through header_tree_node. */
get_local_header_parent_id() {
if (
this.header_tree_node !== undefined &&
this.header_tree_node.parent_ast !== undefined &&
this.header_tree_node.parent_ast.ast !== undefined
) {
return this.header_tree_node.parent_ast.ast.id
}
return undefined
}
/** Works with both actual this.header_tree_node and
* this.header_parent_ids when coming from a database. */
get_header_parent_ids(context) {
const ret = new Set()
const local_parent_id = this.get_local_header_parent_id()
// Refs defined in the current .bigb file + include_path_set
if (local_parent_id !== undefined) {
ret.add(local_parent_id)
}
// Refs not defined from outside in the current .bigb file + include_path_set
// but which were explicitly requested, e.g. we request it for all headers
// to look for external include parents
if (context.options.db_provider) {
const parents_from_db = context.options.db_provider.get_refs_to_as_ids(
REFS_TABLE_PARENT,
this.id,
)
for (const parent_from_db of parents_from_db) {
ret.add(parent_from_db)
}
}
// Refs not defined from outside in the current .bigb file + include_path_set,
// but which were automatically fetched by JOIN our fetch all IDs query:
// we fetch all parent and children via JOIN for every ID we fetch.
// This is needed notably for toplevel scope removal.
for (const header_parent_id of this.header_parent_ids) {
ret.add(header_parent_id)
}
return ret
}
/* Like get_header_parent_ids, but returns the parent AST. */
get_header_parent_asts(context) {
const ret = []
// We replace all parents of the current toplevel with the given one.
// This is what we want for Web.
if (
this.id === context.toplevel_ast.id &&
context.options.parent_id !== undefined
) {
const ast = context.db_provider.get(context.options.parent_id, context)
if (
// Can fail if user passes a parentId that does not exist on Web.
ast !== undefined
) {
ret.push(ast)
}
} else {
const header_parent_ids = this.get_header_parent_ids(context);
for (const header_parent_id of header_parent_ids) {
if (header_parent_id !== undefined) {
ret.push(context.db_provider.get(header_parent_id, context));
}
}
}
return ret
}
is_header_local_descendant_of(ancestor, context) {
let cur_ast = this;
const ancestor_id = ancestor.id;
while (true) {
let cur_id = cur_ast.get_local_header_parent_id()
if (cur_id === undefined) {
return false;
}
cur_ast = context.db_provider.get(cur_id, context)
if (cur_ast.id === ancestor_id) {
return true;
}
}
}
is_last_in_argument() {
return this.parent_argument_index === this.parent_argument.length() - 1
}
isSynonym() {
return this.validation_output.synonym.boolean ||
this.validation_output.synonymNoScope.boolean
}
/** Manual implementation. There must be a better way, but I can't find it... */
static fromJSON(ast_json, context) {
// Post order depth first convert the AST JSON tree.
let new_ast
let ast_head = JSON.parse(ast_json)
const toplevel_arg = []
const todo_visit = [[toplevel_arg, ast_head]];
while (todo_visit.length !== 0) {
const [parent_arg, ast_json] = todo_visit[todo_visit.length - 1];
let finishedSubtrees = false
let done = false
for (const argname in ast_json.args) {
const asts = ast_json.args[argname].asts
if (ast_head === asts[asts.length - 1]) {
finishedSubtrees = true
done = true
break
}
if (done) {
break
}
}
let isLeaf = true
for (const arg_name in ast_json.args) {
const arg_json = ast_json.args[arg_name];
if (arg_json.asts.length > 0) {
isLeaf = false
break
}
}
if (finishedSubtrees || isLeaf) {
todo_visit.pop()
// Visit.
const new_args = {}
for (const arg_name in ast_json.args) {
const arg_json = ast_json.args[arg_name];
const new_arg = new AstArgument(arg_json.asts, arg_json.source_location);
new_args[arg_name] = new_arg
}
new_ast = new AstNode(
AstType[ast_json.node_type],
ast_json.macro_name,
new_args,
ast_json.source_location,
{
text: ast_json.text,
first_toplevel_child: ast_json.first_toplevel_child,
header_tree_node_word_count: ast_json.header_tree_node_word_count,
is_first_header_in_input_file: ast_json.is_first_header_in_input_file,
scope: ast_json.scope,
subdir: ast_json.subdir,
split_default: ast_json.split_default,
// TODO: Remove synonym from JSON, use Ref.type = synonym instead
synonym: ast_json.synonym,
word_count: ast_json.word_count,
}
);
if (context !== undefined) {
validateAst(new_ast, context)
}
parent_arg.push(new_ast)
ast_head = new_ast
} else {
for (const arg_name in ast_json.args) {
const arg_json = ast_json.args[arg_name];
for (const ast_child_json of arg_json.asts.slice().reverse()) {
todo_visit.push([arg_json.asts, ast_child_json]);
}
arg_json.asts.length = 0
}
}
}
return new_ast
}
/** Calculate the output path for this Ast. */
output_path(context, options={}) {
let ast = this
let id
let input_path = ast.source_location.path
if (input_path === undefined || !context.options.db_provider) {
return {}
} else {
if ('effective_id' in options) {
// Used for synonyms.
id = options.effective_id
ast = context.db_provider.get(id, context);
} else {
id = ast.id
}
const ast_undefined = ast === undefined
if (!ast_undefined && ast.macro_name !== Macro.HEADER_MACRO_NAME) {
id = ast.get_header_parent_ids(context).values().next().value;
}
let ast_input_path_toplevel_id
const get_file_ret = context.options.db_provider.get_file(input_path);
if (get_file_ret) {
ast_input_path_toplevel_id = get_file_ret.toplevel_id
} else {
// The only way this can happen is if we are in the current file, and it hasn't
// been added to the file db yet.
ast_input_path_toplevel_id = context.toplevel_id
}
let split_suffix;
if (!ast_undefined && ast.args.splitSuffix !== undefined) {
split_suffix = renderArg(ast.args.splitSuffix, context);
}
const args = {
ast_id: id,
ast_input_path: input_path,
ast_undefined,
context_to_split_headers: context.to_split_headers,
ast_input_path_toplevel_id,
path_sep: context.options.path_sep,
splitDefaultNotToplevel: context.options.ourbigbook_json.h.splitDefaultNotToplevel,
split_suffix,
toSplitHeadersOverride: options.toSplitHeadersOverride,
}
if (!ast_undefined) {
args.ast_is_first_header_in_input_file = ast.is_first_header_in_input_file
args.ast_split_default = ast.split_default
args.ast_toplevel_id = ast.toplevel_id
}
const ret = outputPathBase(args);
if (ret === undefined) {
return {}
} else {
const { dirname, basename, split_suffix: split_suffix_used } = ret
return {
path: pathJoin(dirname, basename + '.' + OUTPUT_FORMATS[context.options.output_format].ext, context.options.path_sep),
dirname,
basename,
split_suffix: split_suffix_used,
}
}
}
}
setup_argument(argname, arg) {
arg.parent_ast = this;
arg.argument_name = argname;
}
// Recursively set source_location on this subtree for elements that
// don't have it yet. Convenient for manually created trees.
set_source_location(source_location) {
const todo_visit_asts = [this]
const todo_visit_args = []
while (todo_visit_asts.length > 0) {
const ast = todo_visit_asts.pop();
if (ast.source_location === undefined) {
ast.source_location = source_location
}
for (const argname in ast.args) {
todo_visit_args.push(ast.args[argname])
}
while (todo_visit_args.length > 0) {
const arg = todo_visit_args.pop();
if (arg.source_location === undefined) {
arg.source_location = source_location
}
for (const ast of arg) {
todo_visit_asts.push(ast)
}
}
}
}
// Set attrs to this AstNode and all its descdendants.
set_recursively(attrs) {
const todo_visit_asts = [this]
const todo_visit_args = []
while (todo_visit_asts.length > 0) {
const ast = todo_visit_asts.pop();
Object.assign(ast, attrs)
for (const argname in ast.args) {
todo_visit_args.push(ast.args[argname])
}
while (todo_visit_args.length > 0) {
const arg = todo_visit_args.pop();
for (const ast of arg) {
todo_visit_asts.push(ast)
}
}
}
}
// A simplified recursive view of the most important fields of this AstNode and children,
// with one AstNode or AstArgument per line. Indispensable for debugging, since the toJSON
// is huge.
toString() {
let ret = []
let indent_marker = ' '
const todo_visit = [{type: 'ast', value: this, indent: 0}]
while (todo_visit.length > 0) {
const thing = todo_visit.pop();
const indent = thing.indent + 1
const indent_str = indent_marker.repeat(thing.indent)
if (thing.type === 'ast') {
const ast = thing.value
let plaintext
if (ast.node_type === AstType.PLAINTEXT) {
plaintext = ` ${JSON.stringify(ast.text)}`
} else {
plaintext = ''
}
let idstr
if (ast.id) {
idstr = ` id="${ast.id}"`
} else {
idstr = ''
}
const pref = `${indent_str}ast `
let typestr
if (ast.node_type === AstType.PLAINTEXT || ast.node_type === AstType.MACRO) {
typestr = `${ast.macro_name}${plaintext}${idstr}`
} else {
typestr = `${ast.node_type.toString()}`
}
ret.push(`${pref}${typestr}`)
if (ast.node_type === AstType.MACRO) {
const args = ast.args
const argnames = Object.keys(args).sort().reverse()
for (const argname of argnames) {
todo_visit.push({type: 'arg', value: args[argname], argname, indent })
}
}
} else {
const arg = thing.value
if (arg) {
ret.push(`${indent_str}arg ${thing.argname}`)
for (const ast of arg.slice().reverse()) {
todo_visit.push({type: 'ast', value: ast, indent })
}
}
}
}
return ret.join('\n')
}
toJSON() {
const ret = {
macro_name: this.macro_name,
node_type: symbolToString(this.node_type),
scope: this.scope,
source_location: this.source_location,
subdir: this.subdir,
text: this.text,
first_toplevel_child: this.first_toplevel_child,
is_first_header_in_input_file: this.is_first_header_in_input_file,
split_default: this.split_default,
// TODO: Remove synonym from JSON, use Ref.type = synonym instead
synonym: this.synonym,
word_count: this.word_count,
}
const args_given = { ...this.args }
for (const argname in this.args) {
if (
argname in this.validation_output &&
!this.validation_output[argname].given
) {
delete args_given[argname]
}
}
ret.args = args_given
if (this.header_tree_node !== undefined) {
ret.header_tree_node_word_count = this.header_tree_node.word_count
}
return ret;
}
}
exports.AstNode = AstNode;
class AstArgument {
/** @param {List[AstNode]} nodes
* @ param {SourceLocation} source_location
*/
constructor(asts, source_location, opts={}) {
const { parent_ast, argument_name } = opts
if (asts === undefined) {
this.asts = []
} else {
this.asts = asts
}
this.source_location = source_location;
// AstNode
this.parent_ast = parent_ast
// String
this.argument_name = argument_name
// boolean
this.has_paragraph = undefined;
this.not_all_block = undefined;
let i = 0;
for (const ast of this.asts) {
ast.parent_argument = this;
ast.parent_argument_index = i;
i++;
};
}
static copyEmpty(arg) {
return new AstArgument(
[],
arg.source_location,
{
argument_name: arg.argument_name,
parent_ast: arg.parent_ast,
}
)
}
concat(...other) {
return this.asts.concat(...other)
}
get(i) {
return this.asts[i]
}
length() {
return this.asts.length
}
map(fn) {
return this.asts.map(fn)
}
set(i, val) {
this.asts[i] = val
}
slice(start, end) {
return new AstArgument(this.asts.slice(start, end), this.source_location)
}
splice(start, deleteCount, ...items) {
return this.asts.splice(start, deleteCount, ...items)
}
reverse() {
this.asts.reverse()
return this
}
push(...new_asts) {
const old_length = this.asts.length;
const ret = this.asts.push(...new_asts);
let i = 0;
for (const ast of new_asts) {
ast.parent_argument = this;
ast.parent_argument = this;
ast.parent_argument_index = old_length + i;
i++;
}
return ret;
}
reset() {
this.asts = []
}
*[Symbol.iterator] () {
for (const v of this.asts) {
yield(v)
}
}
toJSON() {
return {
asts: this.asts,
source_location: this.source_location,
}
}
toString() {
const ret = []
for (const ast of this.asts) {
ret.push(ast.toString())
}
return ret.join('\n')
}
}
class ErrorMessage {
/**
* @param {number} severity 1: most severe, 2: next, 3...
*/
constructor(message, source_location, severity=1) {
this.message = message;
this.source_location = source_location;
this.severity = severity;
}
toString() {
let ret = 'error: ';
let had_line_or_col = false;
if (this.source_location.path !== undefined) {
ret += `${this.source_location.path}:`;
}
if (this.source_location.line !== undefined) {
ret += `${this.source_location.line}`;
had_line_or_col = true;
}
if (this.source_location.column !== undefined) {
if (this.source_location.line !== undefined) {
ret += `:`;
}
ret += `${this.source_location.column}`;
had_line_or_col = true;
}
if (had_line_or_col)
ret += ': ';
ret += this.message;
return ret
}
}
function isAbsoluteXref(id, context) {
return id[0] === Macro.HEADER_SCOPE_SEPARATOR ||
(context.options.x_leading_at_to_web && id[0] === AT_MENTION_CHAR)
}
function resolveAbsoluteXref(id, context) {
if (context.options.ref_prefix) {
return context.options.ref_prefix + id
} else {
return id.substring(1)
}
}
/** Set a context.option that may come from ourbigbook.json.
* This method can also be used from ourbigbook to resolve the values of options,
* in the case of options that have effects on both Library and CLI.
*/
function resolveOption(options, opt) {
let ret = options[opt]
if (ret !== undefined) {
return options[opt]
}
return options.ourbigbook_json[opt]
}
exports.resolveOption = resolveOption
/**
* Interface to retrieving the nodes of IDs defined in external files.
*
* We need the abstraction because IDs will come from widely different locations
* between browser and local Node.js operation:
*
* - browser: HTTP requests
* - local: sqlite database
*/
class DbProvider {
/**
* @return remove all IDs from this ID provider for the given path.
* For example, on a local ID database cache, this would clear
* all IDs from the cache.
*/
clear(input_path_noext_renamed) { throw new Error('unimplemented'); }
/**
* @param {String} id
* @param {String} current_scope: scope node of the location
* from which ID get is being done. This affects the final
* ID obtained due to scope resolution.
* @return {Union[AstNode,undefined]}.
* undefined: ID not found
* Otherwise, the ast node for the given ID
*/
get(id, context, current_scope) {
if (isAbsoluteXref(id, context)) {
return this.get_noscope(resolveAbsoluteXref(id, context), context);
} else {
if (current_scope !== undefined) {
current_scope += Macro.HEADER_SCOPE_SEPARATOR
for (let i = current_scope.length - 1; i > 0; i--) {
if (current_scope[i] === Macro.HEADER_SCOPE_SEPARATOR) {
let pref
if (id === '') {
pref = current_scope.substring(0, i)
} else {
pref = current_scope.substring(0, i + Macro.HEADER_SCOPE_SEPARATOR.length)
}
let resolved_scope_id = this.get_noscope(pref + id, context);
if (resolved_scope_id !== undefined) {
return resolved_scope_id;
}
}
}
}
// Not found with ID resolution, so just try to get the exact ID.
return this.get_noscope(id, context);
}
}
/** Like get, but do not resolve scope. */
get_noscope(id) {
return this.get_noscope_base(id);
}
get_noscope_raw(ids) { throw new Error('unimplemented'); }
get_noscope_base(id) {
return this.get_noscopes_base(new Set([id]))[0]
}
get_noscopes_base(ids, ignore_paths_set) { throw new Error('unimplemented'); }
/** Array[{id: String, defined_at: String}]
*/
get_refs_to(type, to_id, reversed=false) { throw new Error('unimplemented'); }
/**
* Unlike get_refs_to_as_ids, this function deduplicates possible scopes of an ID.
* selecting only the correct one for each.
*
* @param {String} id
* @return {Array[AstNode]}: all header nodes that have the given ID
* as a parent includer.
*/
get_refs_to_as_asts(type, to_id, context, opts={}) {
const { current_scope, reversed } = opts
let ref_ids = this.get_refs_to_as_ids(type, to_id, reversed)
let ret = {};
for (const ref_id of ref_ids) {
const from_ast = this.get(ref_id, context, current_scope);
if (from_ast === undefined) {
if (!context.ignore_errors) {
throw new Error(`could not find reference in database: ${ref_id}`);
}
} else {
ret[from_ast.id] = from_ast;
}
}
return Object.entries(ret).map(kv => kv[1]);
}
/** @return Set[string] the IDs that reference the given AST
*
* The return contains multiple possible refs considering unresolved scopes.
* The final correct scope resolution is not calculated by this function.
**/
get_refs_to_as_ids(type, to_id, reversed=false) {
let other_key
const entries = this.get_refs_to(type, to_id, reversed)
return new Set(entries.map(e => e.id))
}
get_file(path) { throw new Error('unimplemented'); }
async fetch_files(path, context) { throw new Error('unimplemented'); }
}
exports.DbProvider = DbProvider;
/** DbProvider that first tries db_provider_1 and then db_provider_2.
*
* The initial use case for this is to transparently use either IDs defined
* in the current document, or IDs defined externally.
*/
class ChainedDbProvider extends DbProvider {
constructor(db_provider_1, db_provider_2) {
super();
this.db_provider_1 = db_provider_1;
this.db_provider_2 = db_provider_2;
}
get_noscope_base(id) {
let ret = this.db_provider_1.get_noscope_base(id);
if (ret !== undefined) {
return ret;
}
ret = this.db_provider_2.get_noscope_base(id);
if (ret !== undefined) {
return ret;
}
return undefined;
}
get_refs_to(type, to_id, reverse=false) {
return this.db_provider_1.get_refs_to(type, to_id, reverse).concat(
this.db_provider_2.get_refs_to(type, to_id, reverse))
}
get_file(path) {
return this.db_provider_1.get_file(path) || this.db_provider_2.get_file(path)
}
}
/** ID provider from a dict.
* The initial use case is to represent locally defined IDs, and inject
* them into ChainedDbProvider together with externally defined IDs.
*/
class DictDbProvider extends DbProvider {
constructor(dict, refs_to) {
super();
this.dict = dict;
this.refs_to = refs_to;
}
get_noscope_base(id) {
return this.dict[id];
}
get_refs_to(type, to_id, reverse=false) {
const ret = []
const from_ids_reverse = this.refs_to[reverse]
if (from_ids_reverse !== undefined) {
const from_ids_type = from_ids_reverse[to_id];
if (from_ids_type !== undefined) {
const from_ids = from_ids_type[type];
if (from_ids !== undefined) {
for (const from_id in from_ids) {
const defined_ats = from_ids[from_id].defined_at
for (const defined_at in defined_ats) {
ret.push({ id: from_id, defined_at })
}
}
}
}
}
return ret
}
get_file(path) {
return null
}
}
/** Represents possible arguments of each Macro */
class MacroArgument {
/**
* @param {String} name
*/
constructor(options) {
options = { ...options }
if (!('automaticTopicLinks' in options)) {
// If true, automatic topic links can be generated inside this argument,
// so long as count_words is also true.
options.automaticTopicLinks = true;
}
if (!('elide_link_only' in options)) {
// If the only thing contained in this argument is a single
// Macro.LINK_MACRO_NAME macro, AST post processing instead extracts
// the href of that macro, and transforms it into a text node with that href.
//
// Goal: to allow the use to write both \a[http://example.com] and
// \p[http://example.com] and get what a sane person expects, see also:
// https://docs.ourbigbook.com#shorthand-link-parsing-rules
options.elide_link_only = false;
}
if (!('boolean' in options)) {
// https://docs.ourbigbook.com#boolean-argument
options.boolean = false;
}
if (!('cannotContain' in options)) {
// Set of strings, each one is a macro name that cannot be
// recursively contained inside this argument.
options.cannotContain = new Set()
}
if (!('count_words' in options)) {
options.count_words = false;
}
if (!('default' in options)) {
// https://docs.ourbigbook.com#boolean-named-arguments
options.default = undefined;
}
if (!('disabled' in options)) {
// https://docs.ourbigbook.com/disabled-macro-argument
options.disabled = false;
}
if (!('empty' in options)) {
// https://docs.ourbigbook.com/empty-macro-argument
options.empty = false;
}
const name = options.name
let inlineOnly = options.inlineOnly
this.inlineOnly = (inlineOnly === undefined) ? (options.name === Macro.TITLE_ARGUMENT_NAME) : inlineOnly
if (!('mandatory' in options)) {
// https://docs.ourbigbook.com#mandatory-positional-arguments
options.mandatory = false;
}
if (!('multiple' in options)) {
// https://docs.ourbigbook.com#multiple-argument
options.multiple = false;
}
if (!('ourbigbook_output_prefer_literal' in options)) {
options.ourbigbook_output_prefer_literal = false
}
if (!('positive_nonzero_integer' in options)) {
options.positive_nonzero_integer = false;
}
if (!('remove_whitespace_children' in options)) {
// https://docs.ourbigbook.com#remove-whitespace-children
options.remove_whitespace_children = false;
}
if (!('renderLinksAsPlaintext' in options)) {
options.renderLinksAsPlaintext = false
}
this.automaticTopicLinks = options.automaticTopicLinks
this.boolean = options.boolean
this.cannotContain = options.cannotContain
this.count_words = options.count_words
this.disabled = options.disabled
this.empty = options.empty
this.multiple = options.multiple
this.default = options.default
this.elide_link_only = options.elide_link_only
this.mandatory = options.mandatory
this.name = name
this.positive_nonzero_integer = options.positive_nonzero_integer
this.remove_whitespace_children = options.remove_whitespace_children
this.renderLinksAsPlaintext = options.renderLinksAsPlaintext
this.ourbigbook_output_prefer_literal = options.ourbigbook_output_prefer_literal
}
}
class Macro {
/**
* Encapsulates properties of macros, including how to convert
* them to various output formats.
*
* @param {String} name
* @param {Array[MacroArgument]} args
* @param {Function} convert
* @param {Object} options
* {boolean} inline - is this inline content?
* (HTML5 elements that can go in paragraphs). This matters to:
* - determine where `\n\n` paragraphs will split
* - inline content does not get IDs
* {String} auto_parent - automatically surround consecutive sequences of macros with
* the same parent auto_parent into a node with auto_parent type. E.g.,
* to group list items into ul.
* {Set[String]} auto_parent_skip - don't do auto parent generation if the parent is one of these types.
* {Function[AstNode, Object] -> String} get_number - return the number that shows on on full references
* as a string, e.g. "123" in "Figure 123." or "1.2.3" in "Section 1.2.3.".
* A return of undefined means that the number is not available, e.g. this is current limitation
* of internal links to other files (could be implemented).
* {Function[AstNode, Object] -> Bool} macro_counts_ignore - if true, then an ID should not be automatically given
* to this node. This is usually the case for nodes that are not visible in the final output,
* otherwise that would confuse readers.
*/
constructor(name, positional_args, options={}) {
options = { ...options }
if (!('auto_parent' in options)) {
// https://docs.ourbigbook.com#auto_parent
}
if (!('auto_parent_skip' in options)) {
options.auto_parent_skip = new Set();
}
if (!('captionNumberVisible' in options)) {
options.captionNumberVisible = function(ast) { return false; }
}
if (!('caption_prefix' in options)) {
options.caption_prefix = capitalizeFirstLetter(name);
}
if (!('default_x_style_full' in options)) {
options.default_x_style_full = true;
}
if (!('get_number' in options)) {
options.get_number = function(ast, context) { return ast.macro_count_visible; }
}
if (!('get_title_arg' in options)) {
options.get_title_arg = function(ast, context) {
return ast.args[Macro.TITLE_ARGUMENT_NAME];
}
}
if (!('id_prefix' in options)) {
options.id_prefix = titleToId(name);
}
if (!('image_video_content_func' in options)) {
options.image_video_content_func = function() { throw new Error('unimplemented'); };
}
if (!('macro_counts_ignore' in options)) {
// Applications:
// * if an AST node with an ID but no-rendered HTML ID, this breaks editor scroll sync
// Returning true makes it not have an ID at all.
options.macro_counts_ignore = function(ast) {
return false;
}
}
if (!('named_args' in options)) {
options.named_args = [];
}
const inline = options.inline
this.inline = inline === undefined ? false : inline
if (!('show_disambiguate' in options)) {
options.show_disambiguate = false;
}
if (!('source_func' in options)) {
options.source_func = function() { throw new Error('unimplemented'); };
}
if (!('toplevel_link' in options)) {
options.toplevel_link = true;
}
if (!('xss_safe' in options)) {
options.xss_safe = true;
}
if (!('xss_safe_alt' in options)) {
}
this.name = name;
this.positional_args = positional_args;
{
let named_args = {};
for (const arg of options.named_args) {
named_args[arg.name] = arg;
}
this.named_args = named_args;
}
this.auto_parent = options.auto_parent;
this.auto_parent_skip = options.auto_parent_skip;
this.convert_funcs = {}
this.id_prefix = options.id_prefix;
this.options = options;
this.remove_whitespace_children = options.remove_whitespace_children;
this.toplevel_link = options.toplevel_link;
this.name_to_arg = {};
for (const arg of this.positional_args) {
let name = arg.name;
this.check_name(name);
this.name_to_arg[name] = arg;
}
for (const name in this.named_args) {
this.check_name(name);
this.name_to_arg[name] = this.named_args[name];
}
// Add arguments common to all macros.
for (const argname of Macro.COMMON_ARGNAMES) {
this.named_args[argname] = new MacroArgument({
name: argname,
})
this.name_to_arg[argname] = this.named_args[argname];
}
}
add_convert_function(output_format, my_function, macro_name) {
this.convert_funcs[output_format] = my_function;
// This produces incredibly superior render backtraces as you can immediately spot which
// type of macro is being rendered without entering the line numbers.
Object.defineProperty(my_function, 'name', { value: `render_func_${macro_name}` })
}
check_name(name) {
if (Macro.COMMON_ARGNAMES_SET.has(name)) {
throw new Error(`name "${name}" is reserved and automatically added`);
}
if (name in this.name_to_arg) {
throw new Error('name already taken: ' + name);
}
}
toJSON() {
const options = this.options;
const ordered_options = {};
Object.keys(options).sort().forEach(function(key) {
ordered_options[key] = options[key];
});
return {
name: this.name,
options: ordered_options,
positional_args: this.positional_args,
named_args: this.named_args,
}
}
}
exports.Macro = Macro;
Macro.BOLD_MACRO_NAME = 'b'
// Macro names defined here are those that have magic properties, e.g.
// headers are used by the 'toc'.
Macro.OURBIGBOOK_EXAMPLE_MACRO_NAME = 'OurBigBookExample';
Macro.BOOLEAN_ARGUMENT_FALSE = '0'
Macro.BOOLEAN_ARGUMENT_TRUE = '1'
Macro.CODE_MACRO_NAME = 'c';
// Add arguments common to all macros.
Macro.DISAMBIGUATE_ARGUMENT_NAME = 'disambiguate';
Macro.ID_ARGUMENT_NAME = 'id';
// Undocumented argument used only for testing.
// Adds data-X to one of the rendered HTML elements.
// to facilitate XPath selection.
Macro.TEST_DATA_ARGUMENT_NAME = 'ourbigbookTestData';
Macro.TEST_DATA_HTML_PROP = 'data-ourbigbook-test';
Macro.SYNONYM_ARGUMENT_NAME = 'synonym';
Macro.CAPITALIZE_ARGUMENT_NAME = 'c';
Macro.COMMON_ARGNAMES = [
Macro.ID_ARGUMENT_NAME,
Macro.DISAMBIGUATE_ARGUMENT_NAME,
Macro.TEST_DATA_ARGUMENT_NAME,
];
Macro.COMMON_ARGNAMES_SET = new Set(Macro.COMMON_ARGNAMES)
Macro.CONTENT_ARGUMENT_NAME = 'content';
Macro.DESCRIPTION_ARGUMENT_NAME = 'description';
// A simple test-only macro that only has a sane form.
// Because we keep adding shorthand versions of everything,
// adding an shorthand version of \Q which was used extensively
// in tests was a motivation for this.
Macro.TEST_SANE_ONLY = 'TestSaneOnly';
Macro.HEADER_MACRO_NAME = 'H';
Macro.HEADER_CHILD_ARGNAME = 'child';
Macro.HEADER_TAG_ARGNAME = 'tag';
Macro.X_MACRO_NAME = 'x';
Macro.HEADER_SCOPE_SEPARATOR = runtime_common.HEADER_SCOPE_SEPARATOR
Macro.INCLUDE_MACRO_NAME = 'Include';
Macro.LINE_BREAK_MACRO_NAME = 'br'
Macro.LINK_MACRO_NAME = 'a';
Macro.LIST_ITEM_MACRO_NAME = 'L';
Macro.MATH_MACRO_NAME = 'm';
Macro.PASSTHROUGH_MACRO_NAME = 'passthrough'
Macro.PARAGRAPH_MACRO_NAME = 'P';
Macro.PLAINTEXT_MACRO_NAME = 'plaintext';
Macro.QUOTE_MACRO_NAME = 'Q'
Macro.TABLE_MACRO_NAME = 'Table';
Macro.TD_MACRO_NAME = 'Td';
Macro.TH_MACRO_NAME = 'Th';
Macro.TR_MACRO_NAME = 'Tr';
Macro.TITLE_ARGUMENT_NAME = 'title';
Macro.TITLE2_ARGUMENT_NAME = 'title2';
// We set a fixed magic ID to the ToC because:
// - when doing --split-headers, the easy approach is to add the ToC node
// after normal ID indexing has happened, which means that we can't link
// to the ToC as other normal links. And if we could, we would have to worry
// about how to avoid ID duplication
// - only a single ToC ever renders per document. So we can just have a fixed
// magic one.
Macro.RESERVED_ID_PREFIX = '_'
Macro.UNORDERED_LIST_MACRO_NAME = 'Ul';
const FILE_PREFIX = Macro.RESERVED_ID_PREFIX + 'file'
exports.FILE_PREFIX = FILE_PREFIX
Macro.FILE_ID_PREFIX = FILE_PREFIX + Macro.HEADER_SCOPE_SEPARATOR
const RAW_PREFIX = Macro.RESERVED_ID_PREFIX + 'raw'
exports.RAW_PREFIX = RAW_PREFIX
const DIR_PREFIX = Macro.RESERVED_ID_PREFIX + 'dir'
exports.DIR_PREFIX = DIR_PREFIX
Macro.TOC_ID = Macro.RESERVED_ID_PREFIX + 'toc';
Macro.TOC_PREFIX = Macro.TOC_ID + '/'
Macro.TOPLEVEL_MACRO_NAME = 'Toplevel';
/** Helper to create plaintext nodes, since so many of the fields are fixed in that case. */
class PlaintextAstNode extends AstNode {
constructor(text, source_location) {
super(AstType.PLAINTEXT, Macro.PLAINTEXT_MACRO_NAME,
{}, source_location, { text });
}
}
class SourceLocation {
constructor(line, column, path, opts={}) {
this.line = line;
this.column = column;
this.path = path;
this.endLine = opts.endline
this.endColumn = opts.endColumn
}
clone() {
// https://stackoverflow.com/questions/41474986/how-to-clone-a-javascript-es6-class-instance
// Saves about 0.1s out of 2.1s on https://github.com/cirosantilli/cirosantilli.github.io/blob/ed5e39dad5c9ce099b554409d05be0c5c32e5209/ciro-santilli.bigb.
// Noticed on CDT profiling. 5% in one line? I'll take it.
return new SourceLocation(this.line, this.column, this.path)
//return lodash.clone(this);
}
isEqual(other) {
return lodash.isEqual(this, other);
}
toString() {
let ret = []
if (this.path) {
ret.push(this.path + ':')
}
ret.push(`${this.line}:${this.column}`)
return ret.join('')
}
}
exports.SourceLocation = SourceLocation;
class Token {
/**
* @param {String} type
* @param {SourceLocation} source_location
* @param {String} value - Default: undefined
*/
constructor(type, source_location, value) {
this.type = type;
this.source_location = source_location;
this.value = value;
}
toJSON() {
return {
type: this.type.toString(),
source_location: this.source_location,
value: this.value
}
}
}
class Tokenizer {
/**
* @param {String} input_string
*/
constructor(
input_string,
extra_returns={},
show_tokenize=false,
start_line=1,
input_path=undefined
) {
this.chars = Array.from(input_string);
this.cur_c = this.chars[0];
this.source_location = new SourceLocation(start_line, 1, input_path);
this.extra_returns = extra_returns;
this.extra_returns.errors = [];
this.i = 0;
this.in_shorthand_header = false;
this.in_escape_shorthand_link = false;
this.list_level = 0;
this.tokens = [];
this.show_tokenize = show_tokenize;
this.log_debug(`${JSON.stringify(this.chars)}`);
}
/** Advance the current character and set cur_c to the next one.
*
* Maintain the newline count up to date for debug messages.
*
* The current index must only be incremented through this function
* and never directly.
*
* @param {Number} how many to consume
* @return {boolean} true iff we are not reading past the end of the input
*/
consume(n=1) {
for (let done = 0; done < n; done++) {
this.log_debug(`consume ${JSON.stringify(this.cur_c)}`);
if (this.chars[this.i] === '\n') {
if (this.in_shorthand_header) {
this.error(
`shorthand header cannot contain newlines`,
new SourceLocation(
this.source_location.line,
this.source_location.column,
this.source_location.path
)
)
}
this.source_location.line += 1;
this.source_location.column = 1;
} else {
this.source_location.column += 1;
}
this.i += 1;
if (this.i >= this.chars.length) {
this.cur_c = undefined;
return false;
}
this.cur_c = this.chars[this.i];
}
return true;
}
consumeCharRepeatedly(c) {
while (this.cur_c === c) {
this.consume()
}
}
/* Consume list indents, but only when we are closing them, not opening. */
consumeListIndentClose() {
if (this.i === 0 || this.chars[this.i - 1] === '\n') {
let new_list_level = 0;
while (
arrayContainsArrayAt(this.chars, this.i, SHORTHAND_LIST_INDENT) &&
new_list_level < this.list_level
) {
for (const c in SHORTHAND_LIST_INDENT) {
this.consume();
}
new_list_level += 1;
}
if (this.list_level > new_list_level) {
// So that:
// ``
// > asdf
// {title=my title}
// ``
//
// creates a text node 'asdf' and not 'asdf\n'.
this.consume_optional_newline_before_close()
}
for (let i = 0; i < this.list_level - new_list_level; i++) {
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
}
this.list_level = new_list_level;
}
}
consume_plaintext_char() {
return this.plaintext_append_or_create(this.cur_c)
}
/**
* @return {boolean} EOF reached?
*/
consume_optional_newline(literal) {
this.log_debug('consume_optional_newline');
this.log_debug();
if (
!this.is_end() &&
this.cur_c === '\n' &&
(
literal ||
// Shorthand constructs that start with a newline prevent the skip.
(
// Pararaph.
this.peek() !== '\n' &&
// Shorthand start.
this.tokenize_shorthand_start(this.i + 1) === undefined
)
)
) {
this.log_debug();
return this.consume();
}
return true;
}
consume_optional_newline_after_argument() {
if (
!this.is_end() &&
this.cur_c === '\n' &&
!this.in_shorthand_header
) {
const full_indent = SHORTHAND_LIST_INDENT.repeat(this.list_level);
if (
arrayContainsArrayAt(this.chars, this.i + 1, full_indent + START_POSITIONAL_ARGUMENT_CHAR) ||
arrayContainsArrayAt(this.chars, this.i + 1, full_indent + START_NAMED_ARGUMENT_CHAR)
) {
this.consume(full_indent.length + 1);
}
}
}
consume_optional_newline_before_close() {
if (this.tokens.length > 0) {
const last_token = this.tokens[this.tokens.length - 1]
if (last_token.type === TokenType.PLAINTEXT) {
const txt = last_token.value
if (txt[txt.length - 1] === '\n') {
if (last_token.value.length === 1) {
this.tokens.pop()
} else {
last_token.value = txt.substring(0, txt.length - 1);
}
}
}
}
}
error(message, source_location) {
let new_source_location;
if (source_location === undefined) {
new_source_location = new SourceLocation();
} else {
new_source_location = source_location.clone();
}
if (new_source_location.path === undefined)
new_source_location.path = this.source_location.path;
if (new_source_location.line === undefined)
new_source_location.line = this.source_location.line;
if (new_source_location.column === undefined)
new_source_location.column = this.source_location.column;
this.extra_returns.errors.push(
new ErrorMessage(message, new_source_location));
}
is_end() {
return this.i === this.chars.length;
}
log_debug(message='') {
if (this.show_tokenize) {
console.error(`tokenize: ${this.source_location.line}:${this.source_location.column}: ${message}`);
}
}
peek() {
return this.chars[this.i + 1];
}
plaintext_append_or_create(s) {
let new_plaintext = true;
if (this.tokens.length > 0) {
let last_token = this.tokens[this.tokens.length - 1];
if (last_token.type === TokenType.PLAINTEXT) {
last_token.value += s;
new_plaintext = false;
}
}
if (new_plaintext) {
this.push_token(TokenType.PLAINTEXT, s);
}
return this.consume();
}
push_token(token, value, source_location) {
this.log_debug(`push_token ${token.toString()}${value === undefined ? '' : ` ${JSON.stringify(value)}`}`)
let new_source_location;
if (source_location === undefined) {
new_source_location = new SourceLocation();
} else {
new_source_location = source_location.clone();
}
if (new_source_location.line === undefined)
new_source_location.line = this.source_location.line;
if (new_source_location.column === undefined)
new_source_location.column = this.source_location.column;
if (new_source_location.path === undefined)
new_source_location.path = this.source_location.path;
this.tokens.push(new Token(token, new_source_location, value));
}
/**
* @return {Array[Token]}
*/
tokenize() {
// Ignore the last newline of the file.
// It is good practice to always have a newline
// at the end of files, but it doesn't really mean
// that the user wants the last element to contain one.
if (this.chars[this.chars.length - 1] === '\n') {
this.chars.pop();
}
let unterminated_literal = false;
let start_source_location;
while (!this.is_end()) {
this.log_debug(`loop ${JSON.stringify(this.cur_c)}`);
if (this.in_shorthand_header && this.cur_c === '\n') {
this.in_shorthand_header = false;
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
this.consume_optional_newline_after_argument()
}
this.consumeListIndentClose();
start_source_location = this.source_location.clone();
if (this.cur_c === ESCAPE_CHAR) {
this.consume();
if (this.is_end()) {
this.error(`trailing unescaped ${ESCAPE_CHAR}`, start_source_location);
} else if (!charIsIdentifier(this.cur_c)) {
this.consume_plaintext_char();
} else {
// Shorthand link.
for (const known_url_protocol of KNOWN_URL_PROTOCOLS) {
if (arrayContainsArrayAt(this.chars, this.i, known_url_protocol)) {
this.in_escape_shorthand_link = true;
break;
}
}
// Macro.
if (!this.in_escape_shorthand_link) {
let macro_name = this.tokenize_func(charIsIdentifier);
this.consume_optional_newline_after_argument();
this.push_token(
TokenType.MACRO_NAME,
macro_name,
start_source_location,
);
}
}
} else if (this.cur_c === START_NAMED_ARGUMENT_CHAR) {
let source_location = this.source_location.clone();
// Tokenize past the last open char.
let open_length = this.tokenize_func(
c => c === START_NAMED_ARGUMENT_CHAR
).length;
this.push_token(TokenType.NAMED_ARGUMENT_START,
START_NAMED_ARGUMENT_CHAR.repeat(open_length), source_location);
source_location = this.source_location.clone();
if (this.cur_c === undefined) {
// { at the end of file. Test: "named argument: open bracket at end of file fails gracefully".
unterminated_literal = true;
} else {
let arg_name = this.tokenize_func(charIsIdentifier);
this.push_token(TokenType.NAMED_ARGUMENT_NAME, arg_name, source_location);
if (this.cur_c === NAMED_ARGUMENT_EQUAL_CHAR) {
// Consume the = sign.
this.consume();
} else if (this.cur_c === END_NAMED_ARGUMENT_CHAR) {
// Boolean argument.
} else {
this.error(`expected character: '${NAMED_ARGUMENT_EQUAL_CHAR}' or '${END_NAMED_ARGUMENT_CHAR}', got '${this.cur_c}'`);
}
if (open_length === 1) {
this.consume_optional_newline();
} else {
// Literal argument.
let close_string = closingChar(
START_NAMED_ARGUMENT_CHAR).repeat(open_length);
if (!this.tokenize_literal(START_NAMED_ARGUMENT_CHAR, close_string)) {
unterminated_literal = true;
}
this.push_token(TokenType.NAMED_ARGUMENT_END, close_string);
this.consume_optional_newline_after_argument()
}
}
} else if (this.cur_c === END_NAMED_ARGUMENT_CHAR) {
this.consume_optional_newline_before_close();
this.push_token(TokenType.NAMED_ARGUMENT_END, END_NAMED_ARGUMENT_CHAR);
this.consume();
this.consume_optional_newline_after_argument()
} else if (this.cur_c === START_POSITIONAL_ARGUMENT_CHAR) {
let source_location = this.source_location.clone();
// Tokenize past the last open char.
let open_length = this.tokenize_func(
c => c === START_POSITIONAL_ARGUMENT_CHAR
).length;
this.push_token(TokenType.POSITIONAL_ARGUMENT_START,
START_POSITIONAL_ARGUMENT_CHAR.repeat(open_length), source_location);
if (open_length === 1) {
this.consume_optional_newline();
} else {
// Literal argument.
let close_string = closingChar(
START_POSITIONAL_ARGUMENT_CHAR).repeat(open_length);
if (!this.tokenize_literal(START_POSITIONAL_ARGUMENT_CHAR, close_string)) {
unterminated_literal = true;
}
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
this.consume_optional_newline_after_argument()
}
} else if (this.cur_c === END_POSITIONAL_ARGUMENT_CHAR) {
this.consume_optional_newline_before_close();
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
this.consume();
this.consume_optional_newline_after_argument();
} else if (this.cur_c in MAGIC_CHAR_ARGS) {
const source_location = this.source_location.clone();
// Shorthands e.g. $$ math, `` code and <> magic \x.
const open_char = this.cur_c;
const open_length = this.tokenize_func(c => c === open_char).length;
let close_char
let isTopic = false
if (open_char === SHORTHAND_X_START) {
close_char = SHORTHAND_X_END
if (this.cur_c === SHORTHAND_TOPIC_CHAR) {
this.consume(SHORTHAND_TOPIC_CHAR.length)
isTopic = true
}
} else {
close_char = open_char
}
const close_string = close_char.repeat(open_length);
let macro_name = MAGIC_CHAR_ARGS[open_char];
if (open_length > 1) {
macro_name = macro_name.toUpperCase();
}
this.push_token(TokenType.MACRO_NAME, macro_name);
this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
if (!this.tokenize_literal(open_char, close_string, true)) {
unterminated_literal = true;
}
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
if (open_char === SHORTHAND_X_START) {
this.push_token(TokenType.NAMED_ARGUMENT_START, START_NAMED_ARGUMENT_CHAR, source_location);
this.push_token(TokenType.NAMED_ARGUMENT_NAME, isTopic ? 'topic' : 'magic', source_location);
this.push_token(TokenType.NAMED_ARGUMENT_END, END_NAMED_ARGUMENT_CHAR, source_location);
}
this.consume_optional_newline_after_argument()
} else {
if (this.cur_c === '\n') {
// We are either a paragraph or a line break.
this.consume()
// Remove empty lines containing only spaces. Count how many newlines we went through.
let nNewlines = 1
while (true) {
let i = this.i
let nSpaces = 0
if (this.list_level) {
while (this.chars[i] === ' ') {
i++
nSpaces++
}
}
if (this.chars[i] === '\n') {
for (let j = 0; j <= nSpaces; j++) {
this.consume()
}
nNewlines++
} else {
break
}
}
if (nNewlines === 1) {
this.consumeListIndentClose()
this.push_token(TokenType.NEWLINE)
} else if (nNewlines >= 2) {
if (arrayContainsArrayAt(this.chars, this.i, SHORTHAND_LIST_INDENT.repeat(this.list_level) + '\n')) {
// Paragraph comes before list close:
//
// * aa
//
// bb
this.push_token(TokenType.PARAGRAPH)
this.consumeListIndentClose()
} else {
// Paragraph comes after list close:
//
// * aa
//
// bb
this.consumeListIndentClose()
this.push_token(TokenType.PARAGRAPH)
}
if (nNewlines > 2) {
this.error(`paragraph with ${nNewlines} newlines, you must use just two`, new SourceLocation(this.source_location.line - nNewlines + 2, 1))
}
}
continue
}
if (this.cur_c === '\n') {
continue
}
// Shorthand link.
if (this.in_escape_shorthand_link) {
this.in_escape_shorthand_link = false;
} else {
let is_shorthand_link = false;
for (const known_url_protocol of KNOWN_URL_PROTOCOLS) {
if (
arrayContainsArrayAt(this.chars, this.i, known_url_protocol)
) {
const pos_char_after = this.i + known_url_protocol.length;
if (
pos_char_after < this.chars.length &&
!SHORTHAND_LINK_END_CHARS.has(this.chars[pos_char_after])
) {
is_shorthand_link = true;
break;
}
}
}
if (is_shorthand_link) {
this.push_token(TokenType.MACRO_NAME, Macro.LINK_MACRO_NAME);
this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
while (this.consume_plaintext_char()) {
if (SHORTHAND_LINK_END_CHARS.has(this.cur_c)) {
break
}
if (this.cur_c === ESCAPE_CHAR) {
this.consume()
}
}
this.push_token(TokenType.POSITIONAL_ARGUMENT_END)
this.consume_optional_newline_after_argument()
continue
}
}
// Shorthand topic link.
if (this.cur_c === SHORTHAND_TOPIC_CHAR) {
const source_location = this.source_location.clone()
this.push_token(TokenType.MACRO_NAME, Macro.X_MACRO_NAME, source_location)
this.push_token(TokenType.POSITIONAL_ARGUMENT_START, START_POSITIONAL_ARGUMENT_CHAR, source_location)
this.consume(SHORTHAND_TOPIC_CHAR.length)
while (this.consume_plaintext_char()) {
if (SHORTHAND_LINK_END_CHARS.has(this.cur_c)) {
break
}
if (this.cur_c === ESCAPE_CHAR) {
this.consume()
}
}
this.push_token(TokenType.POSITIONAL_ARGUMENT_END, END_POSITIONAL_ARGUMENT_CHAR, source_location)
this.push_token(TokenType.NAMED_ARGUMENT_START, START_NAMED_ARGUMENT_CHAR, source_location)
this.push_token(TokenType.NAMED_ARGUMENT_NAME, 'topic', source_location)
this.push_token(TokenType.NAMED_ARGUMENT_END, END_NAMED_ARGUMENT_CHAR, source_location)
continue
}
// Shorthand lists, tables and quotes.
if (
this.i === 0 ||
this.chars[this.i - 1] === '\n' ||
(
this.tokens.length > 0 &&
(
this.tokens[this.tokens.length - 1].type === TokenType.PARAGRAPH ||
this.tokens[this.tokens.length - 1].type === TokenType.NEWLINE
)
) ||
// Immediately at the start of an argument.
this.tokens.length > 0 && (
this.tokens[this.tokens.length - 1].type === TokenType.NAMED_ARGUMENT_NAME ||
this.tokens[this.tokens.length - 1].type === TokenType.POSITIONAL_ARGUMENT_START
)
) {
let i = this.i;
if (this.cur_c === '\n') {
i += 1;
}
let shorthand_start_return = this.tokenize_shorthand_start(i);
if (shorthand_start_return !== undefined) {
const [shorthand_start, shorthand_start_length] = shorthand_start_return;
if (this.cur_c === '\n') {
this.consume();
}
this.push_token(TokenType.MACRO_NAME, SHORTHAND_STARTS_TO_MACRO_NAME[shorthand_start]);
this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
this.list_level += 1;
this.log_debug(`loop list_level=${this.list_level}`);
for (let i = 0; i < shorthand_start_length; i++) {
this.consume();
}
continue
}
}
// Shorthand headers.
if (
this.i === 0 ||
this.chars[this.i - 1] === '\n'
) {
let i = this.i;
let new_header_level = 0;
while (this.chars[i] === SHORTHAND_HEADER_CHAR) {
i += 1;
new_header_level += 1;
}
if (new_header_level > 0 && this.chars[i] === ' ') {
this.push_token(TokenType.MACRO_NAME, Macro.HEADER_MACRO_NAME);
this.push_token(TokenType.POSITIONAL_ARGUMENT_START, SHORTHAND_HEADER_CHAR.repeat(new_header_level));
this.push_token(TokenType.PLAINTEXT, new_header_level.toString());
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
this.push_token(TokenType.POSITIONAL_ARGUMENT_START);
for (let i = 0; i <= new_header_level; i++)
this.consume();
this.in_shorthand_header = true;
continue
}
}
// Character is nothing else, so finally it is a regular plaintext character.
this.log_debug(`plaintext ${JSON.stringify(this.cur_c)}`);
this.consume_plaintext_char()
}
}
if (unterminated_literal) {
this.error(`unterminated literal argument`, start_source_location);
}
// Close any open headers at the end of the document.
if (this.in_shorthand_header) {
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
}
// Close any open list levels at the end of the document.
for (let i = 0; i < this.list_level; i++) {
this.push_token(TokenType.POSITIONAL_ARGUMENT_END);
}
this.push_token(TokenType.PARAGRAPH);
this.push_token(TokenType.INPUT_END);
return this.tokens;
}
// Create a token with all consecutive chars that are accepted
// by the given function.
tokenize_func(f) {
this.log_debug('tokenize_func');
this.log_debug('this.i: ' + this.i);
this.log_debug(`this.cur_c: '${this.cur_c}'`)
this.log_debug('');
let value = '';
while (f(this.cur_c)) {
value += this.cur_c;
this.consume();
if (this.is_end())
break;
}
return value;
}
/**
* Determine if we are at the start of an shorthand indented sequence
* like an shorthand list '* ' or table '| '
*
* @return {Union[[String,Number],undefined]} -
* - [shorthand_start, length] if any is found. For an empty table or list without space,
* length is shorthand_start.length - 1. Otherwise it equals shorthand_start.length.
* - undefined if none found.
*/
tokenize_shorthand_start(i) {
for (const shorthand_start in SHORTHAND_STARTS_TO_MACRO_NAME) {
if (
arrayContainsArrayAt(this.chars, i, shorthand_start)
) {
// Full shorthand start match.
return [shorthand_start, shorthand_start.length];
}
// Empty table or list without space.
let shorthand_start_nospace = shorthand_start.substring(0, shorthand_start.length - 1);
if (
arrayContainsArrayAt(this.chars, i, shorthand_start_nospace) &&
(
i === this.chars.length - 1 ||
this.chars[i + shorthand_start.length - 1] === '\n'
)
) {
return [shorthand_start, shorthand_start.length - 1];
}
}
return undefined;
}
/**
* Start inside the literal argument after the opening,
* and consume until its end.
*
* @return {boolean} - true if OK, false if unexpected EOF
*/
tokenize_literal(open_char, close_string, openEqualsClose = false) {
this.log_debug('tokenize_literal');
this.log_debug(`this.i: ${this.i}`);
this.log_debug(`open_char: ${open_char}`);
this.log_debug(`close_string ${close_string}`);
this.log_debug('');
if (this.is_end())
return false;
// Remove leading escapes.
let i = this.i;
while (this.chars[i] === ESCAPE_CHAR) {
i++;
if (this.is_end())
return false;
}
if (this.chars[i] === open_char && !openEqualsClose) {
// Skip one of the escape chars if they are followed by an open.
if (!this.consume())
return false;
} else {
if (!this.consume_optional_newline(true))
return false;
}
// Now consume the following unescaped part.
let start_i = this.i;
let start_source_location = this.source_location.clone();
while (
this.chars.slice(this.i, this.i + close_string.length).join('')
!== close_string
) {
if (!this.consume())
return false;
}
// Handle trailing escape.
let append;
let end_i;
if (
this.chars[this.i - 1] === ESCAPE_CHAR &&
this.chars.slice(this.i + 1, this.i + close_string.length + 1).join('') === close_string
) {
// Ignore the trailing backslash.
end_i = this.i - 1;
// Consume the escaped closing char.
if (!this.consume())
return false;
append = closingChar(open_char);
} else {
end_i = this.i;
append = '';
}
// Remove shorthand list indents.
let plaintext = '';
{
let i = start_i;
while (true) {
if (this.chars[i - 1] === '\n') {
if (this.chars[i] === '\n') {
} else if (arrayContainsArrayAt(this.chars, i, SHORTHAND_LIST_INDENT.repeat(this.list_level))) {
i += SHORTHAND_LIST_INDENT.length * this.list_level;
} else {
this.error(`literal argument with indent smaller than current shorthand list`, start_source_location);
}
}
if (i < end_i) {
plaintext += this.chars[i];
} else {
break;
}
i++;
}
}
// Create the token.
this.push_token(
TokenType.PLAINTEXT,
plaintext + append,
start_source_location,
);
this.consume_optional_newline_before_close();
// Skip over the closing string.
for (let i = 0; i < close_string.length; i++)
this.consume();
return true;
}
}
class HeaderTreeNode {
/**
* Structure:
*
* toplevel -> value -> toplevel header ast
* -> child[0] -> value -> h2 1 header ast
* -> parent_ast -> toplevel
* -> child[1] -> value -> h2 2 header ast
* -> parent_ast -> toplevel
*
* P inside h2 1:
* -> parent_ast -> child[0]
* a inside P inside h2 1:
* -> parent_ast -> child[0]
*
* And every non-header element also gets a parent link to its header without child down:
*
* @param {AstNode} value
* @param {HeaderTreeNode} parent_ast
*/
constructor(ast, parent_ast, options={}) {
this.ast = ast;
this.parent_ast = parent_ast;
this.children = [];
this.index = undefined;
this.descendant_count = 0;
this.descendant_word_count = 0;
if (ast !== undefined) {
this.word_count = ast.word_count;
if (!ast.in_header) {
let cur_node = this.parent_ast;
if (cur_node !== undefined && cur_node.parent_ast !== undefined) {
cur_node.update_ancestor_counts(this.word_count + ast.header_tree_node_word_count)
}
}
} else {
this.word_count = 0;
}
}
// TODO how is this different from passing parent_ast on constructor? Forgot.
// Maybe something along the lines of "this allows you to separate creation and chaining".
add_child(child) {
child.index = this.children.length;
this.children.push(child);
let cur_node = this;
while (cur_node !== undefined) {
cur_node.descendant_count += 1;
cur_node = cur_node.parent_ast;
}
}
/** @return {Number} How deep this node is relative to
* the to of the root of the tree. */
get_level() {
let level = 0;
let cur_node = this.parent_ast;
while (cur_node !== undefined) {
level++;
cur_node = cur_node.parent_ast;
}
return level;
}
/** E.g. get number 1.4.2.5 of a Section.
*
* @return {String}
*/
get_nested_number(header_tree_top_level) {
let indexes = [];
let cur_node = this;
while (
// Possible in skipped header levels.
cur_node !== undefined &&
cur_node.ast !== undefined &&
cur_node.ast.numbered &&
cur_node.get_level() !== header_tree_top_level
) {
indexes.push(cur_node.index + 1);
cur_node = cur_node.parent_ast;
}
return indexes.reverse().join('.');
}
toString() {
const ret = [];
let todo_visit;
// False for toplevel of the tree.
if (this.ast === undefined) {
todo_visit = this.children.slice().reverse();
} else {
todo_visit = [this];
}
while (todo_visit.length > 0) {
const cur_node = todo_visit.pop();
const level = cur_node.get_level();
ret.push(`${SHORTHAND_HEADER_CHAR.repeat(level)} h${level} ${cur_node.get_nested_number(1)} ${cur_node.ast.id}`);
todo_visit.push(...cur_node.children.slice().reverse());
}
return ret.join('\n');
}
update_ancestor_counts(add) {
let cur_node = this.parent_ast
while (cur_node !== undefined) {
cur_node.descendant_word_count += add;
cur_node = cur_node.parent_ast;
}
}
}
exports.HeaderTreeNode = HeaderTreeNode
/** Add an entry to the data structures that keep the map of incoming
* and outgoing \x and \x {child} links. */
function addToRefsTo(toid, context, fromid, relation_type, opts={}) {
addToRefsToOneWay(false, toid, context, fromid, relation_type, opts)
addToRefsToOneWay(true, fromid, context, toid, relation_type, opts)
}
function addToRefsToOneWay(reverse, toid, context, fromid, relation_type, opts={}) {
let { child_index, source_location, inflected } = opts
if (inflected === undefined) {
inflected = false
}
let from_to_dict_false = context.refs_to[reverse];
let from_ids;
if (toid in from_to_dict_false) {
from_ids = from_to_dict_false[toid];
} else {
from_ids = {};
from_to_dict_false[toid] = from_ids;
}
let from_ids_relation_type;
if (relation_type in from_ids) {
from_ids_relation_type = from_ids[relation_type]
} else {
from_ids_relation_type = {}
from_ids[relation_type] = from_ids_relation_type
}
let from_ids_relation_type_fromid = from_ids_relation_type[fromid]
if (from_ids_relation_type_fromid === undefined) {
from_ids_relation_type_fromid = { defined_at: {}, child_index }
from_ids_relation_type[fromid] = from_ids_relation_type_fromid
}
let from_ids_relation_type_fromid_defined_at = from_ids_relation_type_fromid.defined_at[context.options.input_path]
if (from_ids_relation_type_fromid_defined_at === undefined) {
from_ids_relation_type_fromid_defined_at = []
from_ids_relation_type_fromid.defined_at[context.options.input_path] = from_ids_relation_type_fromid_defined_at
}
from_ids_relation_type_fromid_defined_at.push({
line: source_location.line,
column: source_location.column,
inflected,
})
}
/**
* Determine if big_array contains small_array starting at index position
* inside the big array.
*
* @return {boolean} true iff if the big array contains the small one
*/
function arrayContainsArrayAt(big_array, position, small_array) {
for (let i = 0; i < small_array.length; i++) {
if (big_array[position + i] !== small_array[i]) {
return false;
}
}
return true;
}
// https://stackoverflow.com/questions/7837456/how-to-compare-arrays-in-javascript
function arrayEquals(arr1, arr2) {
if (arr1.length !== arr2.length)
return false;
for (let i = 0; i < arr1.length; i++) {
if (arr1[i] !== arr2[i])
return false;
}
return true;
}
// Blacklist some overly common English words.
// Sources:
// * https://en.wikipedia.org/wiki/Most_common_words_in_English
// * https://www.ef.co.uk/english-resources/english-vocabulary/top-100-words/
//const AUTOMATIC_TOPIC_LINK_WORD_BLACKLIST = new Set([
// // Don't blacklist these:
// //'day',
// //'people',
// //'time',
// //'two',
// //'year',
// 'a',
// 'about',
// 'above',
// 'across',
// 'against',
// 'all',
// 'along',
// 'also',
// 'am',
// 'among',
// 'an',
// 'and',
// 'any',
// 'anyone',
// 'anything',
// 'around',
// 'as',
// 'at',
// 'be',
// 'because',
// 'before',
// 'behind',
// 'below',
// 'beneath',
// 'beside',
// 'between',
// 'but',
// 'by',
// 'can',
// 'come',
// 'could',
// 'did',
// 'didn-t',
// 'do',
// 'does',
// 'done',
// 'down',
// 'each-other',
// 'each',
// 'either',
// 'even',
// 'everybody',
// 'everyone',
// 'everything',
// 'find',
// 'first',
// 'for',
// 'from',
// 'get',
// 'gets',
// 'gave',
// 'give',
// 'given',
// 'gives',
// 'giving',
// 'go',
// 'got',
// 'had',
// 'has',
// 'have',
// 'he',
// 'her',
// 'here',
// 'hers',
// 'herself',
// 'him',
// 'himself',
// 'his',
// 'how',
// 'i',
// 'if',
// 'in',
// 'into',
// 'is',
// 'it',
// 'its',
// 'itself',
// 'just',
// 'knew',
// 'know',
// 'knows',
// 'like',
// 'look',
// 'made',
// 'make',
// 'makes',
// 'man',
// 'many',
// 'me',
// 'mine',
// 'more',
// 'my',
// 'myself',
// 'near',
// 'neither',
// 'new',
// 'no-one',
// 'no',
// 'nobody',
// 'not',
// 'nothing',
// 'now',
// 'of',
// 'off',
// 'on',
// 'one-another',
// 'one',
// 'only',
// 'or',
// 'other',
// 'our',
// 'ours',
// 'ourselves',
// 'out',
// 'said',
// 'say',
// 'says',
// 'see',
// 'she',
// 'so',
// 'some',
// 'someone',
// 'something',
// 'take',
// 'tell',
// 'than',
// 'that',
// 'the',
// 'their',
// 'theirs',
// 'them',
// 'themselves',
// 'then',
// 'there',
// 'these',
// 'they',
// 'thing',
// 'think',
// 'this',
// 'those',
// 'to',
// 'toward',
// 'under',
// 'up',
// 'upon',
// 'us',
// 'use',
// 'used',
// 'uses',
// 'very',
// 'want',
// 'was',
// 'way',
// 'we',
// 'well',
// 'what',
// 'when',
// 'which',
// 'who',
// 'whom',
// 'whose',
// 'will',
// 'with',
// 'within',
// 'would',
// 'you',
// 'your',
// 'yours',
// 'yourself',
// 'yourselves',
//])
function automaticTopicLinkConsiderId(s) {
//if (s.length === 1) {
// // Catches some crap like:
// // - initials like e.g.
// // - possessive 's
// // - a
// return false
//}
//if (AUTOMATIC_TOPIC_LINK_WORD_BLACKLIST.has(s.toLowerCase())) {
// return false
//}
return true
}
function basename(str, sep) {
return pathSplit(str, sep)[1];
}
/// https://stackoverflow.com/questions/22697936/binary-search-in-javascript/29018745#29018745
function binarySearch(ar, el, compare_fn) {
var m = 0;
var n = ar.length - 1;
while (m <= n) {
var k = (n + m) >> 1;
var cmp = compare_fn(el, ar[k]);
if (cmp > 0) {
m = k + 1;
} else if (cmp < 0) {
n = k - 1;
} else {
return k;
}
}
return -m - 1;
}
function binarySearchInsert(ar, el, compare_fn) {
let index = binarySearch(ar, el, compare_fn);
if (index < 0) {
ar.splice(-(index + 1), 0, el);
}
return ar;
}
function binarySearchLineToIdArrayFn(elem0, elem1) {
return elem0[0] - elem1[0];
}
/** Calculate node ID and add it to the ID index. */
function calculateId(
ast,
context,
non_indexed_ids,
indexed_ids,
macro_counts,
macro_count_global,
macro_counts_visible,
state,
line_to_id_array,
opts={},
) {
const macro_name = ast.macro_name;
const macro = context.macros[macro_name];
const { ignoreEmptyId, isHomeHeader } = opts
// Linear count of each macro type for macros that have IDs.
if (!macro.options.macro_counts_ignore(ast)) {
if (!(macro_name in macro_counts)) {
macro_counts[macro_name] = 0;
}
const macro_count = macro_counts[macro_name] + 1;
macro_counts[macro_name] = macro_count;
ast.macro_count = macro_count;
}
let index_id = true;
let skip_scope = !!(ast.validation_output.synonymNoScope && ast.validation_output.synonymNoScope.boolean)
let idIsEmpty = false
let id
let file_header = ast.macro_name === Macro.HEADER_MACRO_NAME &&
ast.validation_output.file.given
let file_id_text_append
const new_context = cloneAndSet(context, 'id_conversion', true);
const title_arg = macro.options.get_title_arg(ast, context);
const title_text = renderArgNoescape(title_arg, new_context)
// https://docs.ourbigbook.com/#h-file-argument
if (file_header) {
const file_render = renderArg(ast.args.file, new_context)
if (file_render) {
file_id_text_append = file_render
} else {
file_id_text_append = title_text
}
ast.file = file_id_text_append
}
if (isHomeHeader) {
id = TOPLEVEL_INDEX_ID
if (ast.scope !== undefined) {
id = ast.scope
}
const idArg = ast.args[Macro.ID_ARGUMENT_NAME]
if (idArg && idArg.length()) {
parseError(
state,
`the home article cannot have a non-empty "${Macro.ID_ARGUMENT_NAME}" argument as its ID is always empty and the argument is ignored, consider using synonyms instead`,
idArg.source_location
)
}
const disambiguateArg = ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME]
if (disambiguateArg) {
parseError(
state,
`the home article cannot have the "${Macro.DISAMBIGUATE_ARGUMENT_NAME}" explicitly set as its ID is always empty and the argument is ignored`,
disambiguateArg.source_location
)
}
} else {
if (
// This can happen be false for included headers, and this is notably important
// for the toplevel header which gets its ID from the filename.
ast.id === undefined
) {
const macro_id_arg = ast.args[Macro.ID_ARGUMENT_NAME];
if (macro_id_arg === undefined) {
let id_text = '';
const id_prefix = context.macros[ast.macro_name].id_prefix;
if (title_arg !== undefined) {
if (id_prefix !== '') {
id_text += id_prefix + ID_SEPARATOR
}
if (file_header) {
id_text = Macro.FILE_ID_PREFIX + id_text + file_id_text_append
} else {
id_text += titleToIdContext(title_text, new_context.options.ourbigbook_json.id, new_context);
}
const disambiguate_arg = ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME];
if (disambiguate_arg !== undefined) {
id_text += ID_SEPARATOR + titleToId(renderArgNoescape(disambiguate_arg, new_context), new_context.options.ourbigbook_json.id);
}
id = id_text;
} else {
id_text += '_'
}
if (id === undefined) {
index_id = false;
if (!macro.inline) {
const parent_header_tree_node = ast.header_tree_node.parent_ast
if (parent_header_tree_node && new_context.options.prefixNonIndexedIdsWithParentId) {
skip_scope = true
id_text = parent_header_tree_node.ast.id + Macro.HEADER_SCOPE_SEPARATOR + id_text;
}
id_text += macro_count_global;
macro_count_global++
id = id_text;
// IDs of type p-1, p-2, q-1, q-2, etc.
//if (ast.macro_count !== undefined) {
// id_text += ast.macro_count;
// id = id_text;
//}
}
}
} else {
id = renderArgNoescape(macro_id_arg, new_context);
}
if (
index_id &&
id !== undefined &&
id.startsWith(Macro.RESERVED_ID_PREFIX) &&
!file_header
) {
let message = `IDs that start with "${Macro.RESERVED_ID_PREFIX}" are reserved: "${id}"`;
parseError(state, message, ast.source_location);
}
// TODO also reserve 'split'... or use a better convention for it.
if (id === INDEX_BASENAME_NOEXT) {
parseError(
state,
`the ID "${INDEX_BASENAME_NOEXT}" is reserved because it conflicts with HTML index files like index.html: https://github.com/ourbigbook/ourbigbook/issues/337`,
ast.source_location
)
}
if (id === '') {
if (!ignoreEmptyId) {
parseError(state, 'ID cannot be empty except for the home article', ast.source_location);
}
idIsEmpty = true
}
if (id) {
let scope
if (skip_scope) {
scope = new_context.options.ref_prefix
} else if (ast.scope !== undefined) {
scope = ast.scope
}
if (scope) {
id = scope + Macro.HEADER_SCOPE_SEPARATOR + id
}
}
}
}
if (id !== undefined) {
ast.id = id
}
if (!(ignoreEmptyId && idIsEmpty)) {
if (ast.id && ast.subdir && !skip_scope) {
ast.id = ast.subdir + Macro.HEADER_SCOPE_SEPARATOR + ast.id
}
if (file_header && ast.scope) {
const [input_path, ext] = pathSplitext(context.options.input_path)
let inputPathNoRefPrefix = input_path
if (context.options.ref_prefix) {
inputPathNoRefPrefix = inputPathNoRefPrefix.substring(
context.options.ref_prefix.length + 1,
)
}
if (inputPathNoRefPrefix.startsWith(FILE_PREFIX + context.options.path_sep)) {
// This is e.g. for for _file/programming/hello.py
// In that case, the header contents are completely ignored and we just use the filename.
ast.file = inputPathNoRefPrefix.substring(FILE_PREFIX.length + 1)
} else {
// This is to prepend the current directory only in when {file} is used
// inside of a directory e.g.: programming/python.bigb
// = Python
//
// == hello.py
// {file}
//
// but not for _file/programming/hello.py
const [inputDirNoPrefix, basename] = pathSplit(inputPathNoRefPrefix, context.options.path_sep)
if (inputDirNoPrefix.length) {
ast.file = inputDirNoPrefix + Macro.HEADER_SCOPE_SEPARATOR + ast.file
}
}
}
ast.index_id = index_id;
if (ast.id !== undefined && !ast.force_no_index) {
let non_indexed_ast = non_indexed_ids[ast.id];
if (non_indexed_ast === undefined) {
non_indexed_ids[ast.id] = ast;
if (index_id) {
indexed_ids[ast.id] = ast;
const local_id = ast.get_local_header_parent_id()
if (local_id !== undefined) {
addToRefsTo(
ast.id,
context,
local_id,
REFS_TABLE_PARENT,
{
child_index: ast.header_tree_node.index,
source_location: ast.source_location,
}
)
}
}
} else {
const message = duplicateIdErrorMessage(
ast.id,
non_indexed_ast.source_location.path,
non_indexed_ast.source_location.line,
non_indexed_ast.source_location.column
)
parseError(state, message, ast.source_location);
}
if (captionNumberVisible(ast, context)) {
if (!(macro_name in macro_counts_visible)) {
macro_counts_visible[macro_name] = 0;
}
const macro_count = macro_counts_visible[macro_name] + 1;
macro_counts_visible[macro_name] = macro_count;
ast.macro_count_visible = macro_count;
}
binarySearchInsert(line_to_id_array,
[ast.source_location.line, ast.synonym || ast.id], binarySearchLineToIdArrayFn);
}
}
return { idIsEmpty, title_text, macro_count_global }
}
/* Calculate the length of the scope of a child header given its parent ast. */
function calculateScopeLength(parent_ast) {
if (parent_ast !== undefined) {
let scope = parent_ast.calculate_scope();
if (scope !== undefined) {
return scope.length + 1;
}
}
return 0;
}
function cannotPlaceXInYErrorMessage(x, y) {
return `cannot place \\${x} inside of \\${y}`
}
function capitalizeFirstLetter(string) {
return string.charAt(0).toUpperCase() + string.slice(1);
}
exports.capitalizeFirstLetter = capitalizeFirstLetter
function captionNumberVisible(ast, context) {
return ast.index_id || context.macros[ast.macro_name].options.captionNumberVisible(ast, context);
}
function charIsAlphanumeric(c) {
let code = c.codePointAt(0);
return (
// 0-9
(code > 47 && code < 58) ||
// A-Z
(code > 64 && code < 91) ||
// a-z
(code > 96 && code < 123)
)
}
// Valid macro name / argument characters.
// Compatible with JavaScript-like function names / variables.
// https://docs.ourbigbook.com#macro-identifier
function charIsIdentifier(c) {
return charIsAlphanumeric(c)
};
/** Shallow clone an object, and set a given value on the cloned one. */
function cloneAndSet(obj, key, value) {
let new_obj = {...obj};
new_obj[key] = value;
return new_obj;
}
exports.cloneAndSet = cloneAndSet;
function cloneObject(obj) {
// https://stackoverflow.com/questions/41474986/how-to-clone-a-javascript-es6-class-instance
return Object.assign(Object.create(Object.getPrototypeOf(obj)), obj);
}
function closingChar(c) {
if (c === START_POSITIONAL_ARGUMENT_CHAR)
return END_POSITIONAL_ARGUMENT_CHAR;
if (c === START_NAMED_ARGUMENT_CHAR)
return END_NAMED_ARGUMENT_CHAR;
throw new Error('char does not have a close: ' + c);
}
function closingToken(token) {
if (token === TokenType.POSITIONAL_ARGUMENT_START)
return TokenType.POSITIONAL_ARGUMENT_END;
if (token === TokenType.NAMED_ARGUMENT_START)
return TokenType.NAMED_ARGUMENT_END;
throw new Error('token does not have a close: ' + token);
}
function decapitalizeFirstLetter(string) {
return string.charAt(0).toLowerCase() + string.slice(1);
}
exports.decapitalizeFirstLetter = decapitalizeFirstLetter
/**
* Main ourbigbook input to HTML/LaTeX/etc. output JavaScript API.
*
* The CLI interface basically just feeds this.
*
* @param {Object} options
* {DbProvider} external_ids
* {Function[String] -> [string,string]} read_include(id) -> [file_name, content]
* {Number} h_include_level_offset - add this offset to the levels of every header
* {boolean} render - if false, parse the input, but don't render it,
* and return undefined.
* The initial use case for this is to allow a faster and error-less
* first pass when building an entire directory with internal cross file
* references to extract IDs of each file.
* @param {Object} extra_returns
* {Object} key: output_path
* value: Object:
* title: rendered toplevel inner title
* full: full render, including title and body
* This is needed primarily because --html-spit-header
* generates multiple output files for a single input file.
* @return {String}
*/
async function convert(
input_string,
options,
extra_returns={},
) {
let context = convertInitContext(options, extra_returns);
// Tokenize.
let sub_extra_returns;
sub_extra_returns = {};
perfPrint(context, 'tokenize_pre')
const trailingNewlines = input_string.match(/\n\n+$/)
if (trailingNewlines) {
input_string = input_string.replace(/\n\n+$/, '')
}
const tokenizer = new Tokenizer(
input_string,
sub_extra_returns,
context.options.log.tokenize,
context.options.start_line,
context.options.input_path,
)
let tokens = tokenizer.tokenize()
if (trailingNewlines) {
context.errors.push(
new ErrorMessage(
`the document cannot end in two or more newlines`,
new SourceLocation(tokenizer.source_location.line + 1, 1)
)
)
}
if (context.options.log.tokens) {
console.error('tokens:')
for (const t of tokens) {
const s = t.source_location
console.error(`${s.path}:${s.line}:${s.column}: ${t.type.description}${t.value === undefined ? '' : ` ${JSON.stringify(t.value)}`}`)
}
console.error()
}
perfPrint(context, 'tokenize_post')
if (context.options.log['tokens-inside']) {
console.error('tokens:');
for (let i = 0; i < tokens.length; i++) {
console.error(`${i}: ${JSON.stringify(tokens[i], null, 2)}`);
}
console.error();
}
extra_returns.tokens = tokens;
extra_returns.errors.push(...sub_extra_returns.errors);
sub_extra_returns = {};
// Setup context.media_provider_default based on `default-for`.
{
const media_providers = context.options.ourbigbook_json['media-providers'];
context.media_provider_default = {};
for (const media_provider_name in media_providers) {
const media_provider = media_providers[media_provider_name];
if ('default-for' in media_provider) {
for (const default_for of media_provider['default-for']) {
if (default_for[0] == default_for[0].toUpperCase()) {
context.errors.push(new ErrorMessage(`default-for names must start with a lower case letter`, new SourceLocation(1, 1)));
} else {
if (default_for === 'all') {
for (const macro_name of MACRO_WITH_MEDIA_PROVIDER) {
context.media_provider_default[default_for] = media_provider_name;
context.media_provider_default[capitalizeFirstLetter(default_for)] = media_provider_name;
}
} else {
if (MACRO_WITH_MEDIA_PROVIDER.has(default_for)) {
if (context.media_provider_default[default_for] === undefined) {
context.media_provider_default[default_for] = media_provider_name;
context.media_provider_default[capitalizeFirstLetter(default_for)] = media_provider_name;
} else {
context.errors.push(new ErrorMessage(`multiple media providers set for macro "${default_for}"`, new SourceLocation(1, 1)));
}
} else {
context.errors.push(new ErrorMessage(`macro "${default_for}" does not accept media providers`, new SourceLocation(1, 1)));
}
}
}
}
}
}
for (const macro_name of MACRO_WITH_MEDIA_PROVIDER) {
if (context.media_provider_default[macro_name] === undefined) {
context.media_provider_default[macro_name] = 'local';
context.media_provider_default[capitalizeFirstLetter(macro_name)] = 'local';
}
}
}
const toplevel_ast = await parse(tokens, context.options, context, sub_extra_returns);
if (context.options.log['ast-inside']) {
console.error('ast-inside:')
console.error(JSON.stringify(toplevel_ast, null, 2))
console.error()
}
if (context.options.log['ast-inside-simple']) {
console.error('ast-inside-simple:')
console.error(toplevel_ast.toString())
console.error()
}
extra_returns.ast = toplevel_ast;
extra_returns.context = context;
extra_returns.ids = sub_extra_returns.ids;
Object.assign(extra_returns.debug_perf, sub_extra_returns.debug_perf);
extra_returns.errors.push(...sub_extra_returns.errors);
let output;
if (context.options.render) {
// Gather toplevel ids and their asts.
const toplevel_ids = {}
const content = toplevel_ast.args.content;
let first_toplevel_id
let undefined_asts = []
for (const child_ast of content) {
if (child_ast.toplevel_id === undefined) {
/** Some undefined toplevel_ids are unescapable e.g. input from stdin.
*
* But there is on case that we could improve:
*
* ``
* aa
*
* = Bb
* ``
*
* aa is undefined toplevel_id. We are just semi hacking it and grouping all
* undefined toplevel_id with the first defined one.
* */
undefined_asts.push(child_ast)
} else {
let toplevel_ids_entry = toplevel_ids[child_ast.toplevel_id]
if (first_toplevel_id === undefined) {
first_toplevel_id = child_ast.toplevel_id
}
if (toplevel_ids_entry === undefined) {
if (undefined_asts.length) {
toplevel_ids_entry = undefined_asts
undefined_asts = []
} else {
toplevel_ids_entry = []
}
toplevel_ids[child_ast.toplevel_id] = toplevel_ids_entry
}
toplevel_ids_entry.push(child_ast)
}
}
// Nosplit renders. \H toplevel is split here.
for (const toplevel_id in toplevel_ids) {
// TODO this prevents toplevel arguments (currently only {title}) from working. Never used them though.
// The Toplevel element is a bit useless given this setup.
const ret = renderAstList({
asts: toplevel_ids[toplevel_id],
context,
first_toplevel: toplevel_id === first_toplevel_id,
split: false,
});
if (toplevel_id === first_toplevel_id) {
output = ret
}
}
if (undefined_asts.length) {
// There are no headers.
output = renderAstList({
asts: undefined_asts,
context,
split: false,
});
}
perfPrint(context, 'split_render_pre')
// Split header conversion.
if (context.options.split_headers) {
// Reset katex_macros to the ourbigbook.json defaults, otherwise
// macros will get redefined in the split render, as they are redefined on every render.
context.katex_macros = { ...context.options.katex_macros }
const content = toplevel_ast.args.content;
// Gather up each header (must be a direct child of toplevel)
// and all elements that belong to that header (up to the next header).
let asts = [];
let header_count = 0
for (const child_ast of content) {
const macro_name = child_ast.macro_name;
if (
macro_name === Macro.HEADER_MACRO_NAME &&
// Just ignore extra added include headers, these
// were overwriting index-split.html output.
!child_ast.from_include &&
!child_ast.isSynonym()
) {
renderAstList({ asts, context, header_count, split: true });
asts = [];
header_count++
}
asts.push(child_ast);
}
renderAstList({ asts, context, header_count, split: true });
// Because the following conversion would redefine them.
}
perfPrint(context, 'render_post')
if (extra_returns.errors.length === 0) {
// Only add render errors if there are no parse errors before. They could be just a bunch of noise.
extra_returns.errors.push(...context.errors)
}
}
// Sort errors that might have been produced on different conversion
// stages by line.
extra_returns.errors = extra_returns.errors.sort((a, b)=>{
if (a.source_location.path !== undefined && b.source_location.path !== undefined) {
let ret = a.source_location.path.localeCompare(b.source_location.path);
if (ret !== 0)
return ret;
}
if (a.severity < b.severity)
return -1;
if (a.severity > b.severity)
return 1;
if (a.source_location.line < b.source_location.line)
return -1;
if (a.source_location.line > b.source_location.line)
return 1;
if (a.source_location.column < b.source_location.column)
return -1;
if (a.source_location.column > b.source_location.column)
return 1;
return 0;
});
if (output !== undefined) {
if (output[output.length - 1] !== '\n') {
output += '\n';
}
}
perfPrint(context, 'end_convert')
return output;
}
exports.convert = convert;
/** Convert an argument to an XSS-safe output string.
*
* An argument contains a list of nodes, loop over that list of nodes,
* converting them to strings and concatenate all strings.
*
* @param {AstArgument} arg
* @return {String} empty string if arg is undefined
*/
function renderArg(arg, context) {
let converted_arg = [];
if (arg !== undefined) {
if (context.options.output_format === OUTPUT_FORMAT_OURBIGBOOK) {
// TODO should be done in parse not render. And should be done for all output formats.
for (const ast of arg) {
if (ast.macro_name === Macro.PARAGRAPH_MACRO_NAME) {
arg.has_paragraph = true
}
if (
// Can fail for leftover Newline on:
// \H[2][ab
// cd]
ast.macro_name &&
context.macros[ast.macro_name].inline
) {
arg.not_all_block = true
}
}
}
for (const ast of arg) {
converted_arg.push(ast.render(context));
}
}
return converted_arg.join('');
}
exports.renderArg = renderArg
/* Similar to renderArg, but used for IDs.
*
* Because IDs are used programmatically in ourbigbook, we don't escape
* HTML characters at this point.
*
* @param {AstArgument} arg
* @return {String}
*/
function renderArgNoescape(arg, context={}) {
return renderArg(arg, cloneAndSet(context, 'html_escape', false));
}
/** We want HOME_MARKER to be used in some contexts but not others. Notably
* we currently don't want it on local link to toplevel. This may change however:
* https://github.com/ourbigbook/ourbigbook/issues/334
*/
function renderTitlePossibleHomeMarker(ast, context) {
if (
// Not ideal, but I'm not smart enough to factor them out.
// On web, nAncestors doesn't work, and I'm weary of pulling more data in if I can avoid it.
// It is expected that the resolution of https://github.com/ourbigbook/ourbigbook/issues/334
// might remove the need for this by forcing the toplevel ID of the toplevel index to have empty ID,
// and then just set any title as a synonym to it effectively.
context.options.webMode
) {
if (ast.id === context.options.ref_prefix) {
return HTML_HOME_MARKER
}
} else {
if (ast.ancestors(context).length === 0) {
return HTML_HOME_MARKER
}
}
return xTextBase(ast, context).innerWithDisambiguate
}
/* Convert a list of asts.
*
* The simplest type of conversion is to convert
* a single Toplevel element all the way down.
*
* This function adds a dummy Toplevel to a list of AstNodes, and
* sets up some other related stuff to convert that list.
*
* Applications for this include:
* - --split-headers
* - \H toplevel argument
*
* @param {List[Ast]} asts
* @param {boolean} first_toplevel
* @param {Number} header_count
* @param {boolean} split
*/
function renderAstList({ asts, context, first_toplevel, header_count, split }) {
if (
// Can fail if:
// * the first thing in the document is a header
// * the document has no headers
asts.length > 0 &&
!(
context.options.ourbigbook_json.h.splitDefaultNotToplevel &&
header_count === 1
)
) {
context = { ...context }
const options = { ...context.options }
context.options = options;
const first_ast = cloneObject(asts[0]);
if (!first_toplevel) {
// When not in simple header mode, we always have a value-less node, with
// children with values. Now things are a bit more complicated, because we
// want to keep the header tree intact, but at the same time also uniquely point
// to one of the headers. So let's fake a tree node that has only one child we care
// about. And the child does not have this fake parent to be able to see actual parents.
context.header_tree = new HeaderTreeNode();
// Clone this, because we are going to modify it, and it would affect
// non-split headers and final outputs afterwards.
first_ast.header_tree_node = cloneObject(first_ast.header_tree_node);
context.header_tree.add_child(first_ast.header_tree_node);
context.header_tree_top_level = first_ast.header_tree_node.get_level();
}
const output_path_ret = first_ast.output_path(
cloneAndSet(context, 'to_split_headers', split)
)
const { path: output_path, split_suffix } = output_path_ret
if (options.log['split-headers']) {
console.error(`split-headers${split ? '' : ' nosplit'}: ` + output_path);
}
context.toplevel_output_path = output_path;
if (output_path !== undefined) {
const [toplevel_output_path_dir, toplevel_output_path_basename] =
pathSplit(output_path, context.options.path_sep);
context.toplevel_output_path_dir = toplevel_output_path_dir;
} else {
context.toplevel_output_path_dir = '';
}
context.toplevel_ast = first_ast;
const ast_toplevel = new AstNode(
AstType.MACRO,
Macro.TOPLEVEL_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(asts, first_ast.source_location)
},
first_ast.source_location,
);
let toplevel_id
if (context.options.toplevelId) {
toplevel_id = context.options.toplevelId
} else {
toplevel_id = first_ast.id
}
context.toplevel_id = toplevel_id
context.in_split_headers = split;
let rendered_outputs_entry = {}
if (output_path !== undefined) {
// root_relpath
options.template_vars = { ...options.template_vars }
const new_root_relpath = getRootRelpath(output_path, context)
context.root_relpath_shift = path.relative(
options.template_vars.root_relpath,
new_root_relpath
)
options.template_vars.root_relpath = new_root_relpath
options.template_vars.raw_relpath = path.join(new_root_relpath, RAW_PREFIX)
options.template_vars.file_relpath = path.join(new_root_relpath, FILE_PREFIX)
options.template_vars.dir_relpath = path.join(new_root_relpath, DIR_PREFIX)
options.template_vars.file_relpath = path.join(new_root_relpath, FILE_PREFIX)
options.template_vars.toplevel_id = toplevel_id
context.extra_returns.rendered_outputs[output_path] = rendered_outputs_entry
}
// Do the conversion.
context.toc_was_rendered = false
if (!context.options.webMode) {
// We set this so that on local conversion the project toplevel header
// will actually render as is rather than HOME.
// This is horrendous, but I don't know how to make it work otherwise.
// Things get a bit complicated on web, where we want HOME to go everywhere,
// and where title is stored in the database and used everywhere in places
// such as index and so on.
context.renderFirstHeaderNotAsHome = true
}
const ret = ast_toplevel.render(context)
if (output_path !== undefined) {
rendered_outputs_entry.full = ret
rendered_outputs_entry.split = split
rendered_outputs_entry.header_ast = first_ast
rendered_outputs_entry.split_suffix = split_suffix
if (
options.renderH2 &&
first_ast.macro_name === Macro.HEADER_MACRO_NAME
) {
options.h_show_split_header_link = false
const parent_ast = first_ast.get_header_parent_asts(context)[0];
context.header_tree = new HeaderTreeNode();
if (parent_ast) {
context.toplevel_ast = parent_ast;
const output_path_ret = parent_ast.output_path(
cloneAndSet(context, 'to_split_headers', split)
)
const { path: output_path, split_suffix } = output_path_ret
// Hax. It has to be both undefined, and different than the correct output_path.
// I think this is needed because is a slightly different rendering than what
// is ever done outside of web (it is an h2, and its h1 is on another page)
// so it is likely either this hack or another flag.
context.toplevel_output_path = output_path + 'asdf';
}
const header_tree_h1 = new HeaderTreeNode(parent_ast, context.header_tree);
context.header_tree.add_child(header_tree_h1);
first_ast.header_tree_node = cloneObject(first_ast.header_tree_node);
first_ast.header_tree_node.parent_ast = header_tree_h1
first_ast.first_toplevel_child = false
//header_tree_h1.add_child(first_ast.header_tree_node)
context.skipOutputEntry = true
context.forceHeadersHaveTocLink = true
context.toplevel_id = header_tree_h1.id
first_ast.toplevel_id = header_tree_h1.id
rendered_outputs_entry.h2Render = first_ast.render(context)
}
}
return ret
}
}
/**
* @param {Object} options:
* - {Number} start_line
* - {Array} errors
* @return {AstArgument}*/
async function parseInclude(
input_string,
convert_options,
cur_header_level,
input_path,
href,
options={}
) {
convert_options = { ...convert_options }
convert_options.from_include = true;
convert_options.h_parse_level_offset = cur_header_level;
convert_options.input_path = input_path;
convert_options.render = false;
convert_options.toplevel_id = href;
convert_options.header_tree_stack = new Map(convert_options.header_tree_stack);
convert_options.header_tree_id_stack = new Map(convert_options.header_tree_id_stack);
if (options.start_line !== undefined) {
convert_options.start_line = options.start_line;
}
const convert_extra_returns = {};
await convert(
input_string,
convert_options,
convert_extra_returns,
);
if (options.errors !== undefined) {
options.errors.push(...convert_extra_returns.errors);
}
return convert_extra_returns.ast.args.content;
}
// Convert an argument as an ID, notably:
// - no HTML escapes
// - plaintext conversion
function convertIdArg(arg, context) {
return renderArgNoescape(arg,
// This was added because it was blowing up in the edge case of
// \x[\m[1]] and others during parse to setup the x DB
// because we hadn't validate validated elements
// there yet. Not sure we could, no patience.
// This fix relies on the expectation that id_conversion will
// not rely on validateAst. Maybe that is reasonable.
cloneAndSet(context, 'id_conversion', true))
}
/** Resolve input options into their final values considering defaults and
* forcing across values.
*/
function convertInitOptions(options) {
const newOptions = lodash.cloneDeep(OPTION_DEFAULTS)
for (const key in options) {
const val = options[key]
if (val !== undefined) {
newOptions[key] = val
}
}
const optionsOrig = options
options = newOptions
// TODO this fails several --embed-includes tests. Why.
// Futhermore, this would print the exact same for both according to diff.
//console.log(`options: ${require('util').inspect(options, { depth: null })}`)
// The only difference in behavior is that lodash makes copies of all subobjects.
//options = lodash.merge({}, OPTION_DEFAULTS, options)
// TODO get rid of ourbigbook_json this, move everything into options directly.
options.ourbigbook_json = convertInitOurbigbookJson(options.ourbigbook_json)
if (options.publish && 'publishOptions' in options.ourbigbook_json) {
lodash.merge(options, options.ourbigbook_json.publishOptions)
}
if (!('add_test_instrumentation' in options)) { options.add_test_instrumentation = false; }
if (!('automaticTopicLinksMaxWords' in options)) {
// 0 means turned off.
// Values > 0 produce more and more complete results with longer matches
// at the cost of greater computational usage.
// TODO one day maybe add -1.
// https://github.com/ourbigbook/ourbigbook/issues/356
options.automaticTopicLinksMaxWords = 0
}
if (!('body_only' in options)) { options.body_only = false; }
if (!('db_provider' in options)) { options.db_provider = undefined; }
if (!('fixedScopeRemoval' in options)) {
// Rather than removing scopes from children page in a toplevel page that has a scope,
// remove fixed n chars from every single ID. This is used on Web to remove @ from links
// with dynamic article tree.
options.fixedScopeRemoval = undefined;
}
if (!('renderH2' in options)) { options.renderH2 = false; }
if (options.embed_includes) {
// This is bad, but I don't have the time to fix it.
// Embed-includes just doesn't produce a correct ref DB currently.
// https://github.com/ourbigbook/ourbigbook/issues/343
options.ourbigbook_json.lint.filesAreIncluded = false
}
// Check if file exists.
if (!('fs_exists_sync' in options)) { options.fs_exists_sync }
if (!('forbid_include' in options)) {
// If given, must be a string, and is an error given \\Include is used.
options.forbid_include = undefined;
}
if (!('forbid_multi_h1' in options)) {
// Only allow 1 h1 per input source.
// This is also effectively enforced by filesAreIncluded,
// but with a less clear error message.
options.forbid_multi_h1 = false
}
if (!('forbid_multiheader' in options)) {
// Input can only contain a single header.
// If given, must be the string explaining the error.
options.forbid_multiheader = undefined;
}
if (!('from_include' in options)) { options.from_include = false; }
if (!('from_ourbigbook_example' in options)) { options.from_ourbigbook_example = false; }
if (!('auto_generated_source' in options)) {
// true if the input was auto-generated rather than coming
// from a hand written .bigb input file as usual. Initial application:
// don't show "source code of this page" on templates.
options.auto_generated_source = false;
}
if (!('html_embed' in options)) { options.html_embed = false; }
if (!('hFileShowLarge' in options)) { options.hFileShowLarge = false }
if (!('h_parse_level_offset' in options)) {
// When parsing, start the first header at this offset instead of h1.
// This is used when doing includes, since the included header is at.
// an offset relative to where it is included from.
options.h_parse_level_offset = 0;
}
if (!('h_show_split_header_link' in options)) {
options.h_show_split_header_link = true;
}
if (!('h_web_ancestors' in options)) {
// If true, reserve an empty metadata line for web ancestors, which are dynamically loaded.
options.h_web_ancestors = false;
}
if (!('h_web_metadata' in options)) {
// If true, reserve an empty metadata line for web injected elements
// such as like count and date modified.
options.h_web_metadata = false;
}
if (!('h1Only' in options)) {
// Only parse the first element inside toplevel, assumed to be the unique H1 of the file.
// This is a minor performance optimization.
// This option was added to be able to decide the path to be used
// in conversion from header properties a little bit faster.
options.h1Only = false
}
if (!('input_path' in options)) { options.input_path = undefined; }
if (!('internalLinkMetadata' in options)) { options.internalLinkMetadata = false }
if (!('katex_macros' in options)) { options.katex_macros = {}; }
if (!('logoPath' in options)) { options.logoPath = undefined; }
if (!('prefixNonIndexedIdsWithParentId' in options)) {
// E.g. the first paragraph of a header would have ID `_1` without this.
// This this option it becomes instead `header-id/_1`.
options.prefixNonIndexedIdsWithParentId = false;
}
if (!('log' in options)) { options.log = {}; }
if (!('outfile' in options)) {
// Override the default calculated output file for the main input.
options.outfile = undefined;
}
if (!('outdir' in options)) {
// Path of the output directory relative to the toplevel directory.
// E.g. out/html
options.outdir = undefined;
}
if (!('output_format' in options)) { options.output_format = OUTPUT_FORMAT_HTML; }
if (!('path_sep' in options)) { options.path_sep = undefined; }
if (!('parent_id' in options)) {
// Marks the given ID as a parent of the toplevel header of this conversion.
// More precisely, this creates a de-facto virtual Ref from that ID to the one
// generated by this conversion.
// This was introduced for Web, where we have an explicit parentId provided
// at initial conversion time. Without this options, we would need a separate
// conversion to determine the toplevel ID of this file.
options.parent_id = undefined;
}
if (!('read_include' in options)) { options.read_include = () => undefined; }
if (!('read_file' in options)) { options.read_file = () => undefined; }
if (!('ref_prefix' in options)) {
// This option started as a hack as a easier to implement workaround for:
// https://github.com/ourbigbook/ourbigbook/issues/229
// to allow tagged and incoming links to work at all on OurBigBook Web.
// That specific usage should be removed.
//
// However, we later found another usage for it, which should not be removed:
// it is necessary to resolve absolute references like \x[/top-id] correctly to
// \x[@username/top-id] in Web, where it would be set to '@username'.
//
// Not sure if this was intended or not, but now it is also getting used as a
// "detect if ID is a virtual toplevel or not".
options.ref_prefix = TOPLEVEL_INDEX_ID;
}
if (!('render' in options)) { options.render = true; }
if (!('render_include' in options)) {
// If false, \\Include are removed from the rendered output.
// Their side effects such as determining the header tree are still used.
options.render_include = true;
}
if (!('render_metadata' in options)) {
// Render article "metadata" such as: ToC, tagged, incoming links, ancestors.
// This is notable disabled on Web, where metadata is fetched on the fly at page load.
options.render_metadata = true;
}
if (!('start_line' in options)) { options.start_line = 1; }
if (!('show_descendant_count' in options)) { options.show_descendant_count = true; }
if (!('split_headers' in options)) {
options.split_headers = false;
}
if (!('template' in options)) { options.template = undefined; }
if (!('template_scripts_relative' in options)) {
// Like template_styles_relative but for sripts.
options.template_scripts_relative = [];
}
if (!('template_styles_relative' in options)) {
// CSS styles relative to ourbigbook.json. Must be resolved by ourbigbook.convert.
// because of split headers. The relative path expanded result gets prepended
// to `options.template_vars.style`.
options.template_styles_relative = [];
}
if ('template_vars' in options) {
options.template_vars = { ...options.template_vars }
} else {
options.template_vars = {};
}
if (!('head' in options.template_vars)) { options.template_vars.head = ''; }
if (!('root_relpath' in options.template_vars)) { options.template_vars.root_relpath = ''; }
if (!('post_body' in options.template_vars)) { options.template_vars.post_body = ''; }
if (!('style' in options.template_vars)) { options.template_vars.style = ''; }
if (!'toplevelId' in options) {
options.toplevelId = undefined
}
if (!(Macro.TITLE_ARGUMENT_NAME in options)) {
//options[Macro.TITLE_ARGUMENT_NAME] = undefined
}
if (!('x_absolute' in options)) {
// Make all internal links absolute from website root.
// This is the only way that we can have a single rendering that works on both
// /go/topic/<topic> and /username/<topic>. It will also remove the need for
// the ../ hack we were using to make the same links work from both index /username
// and /username/<topic>.
options.x_absolute = false;
}
if (!('tocIdPrefix' in options)) { options.tocIdPrefix = ''; }
if (!('webMode' in options)) {
// Previously we put some changes under more specific options, e.g.
// h_web_ancestors and h_web_metadata, but that was likely overgeneralization,
// let's just dump every web variant under here from now on unless there is a
// specific reason not to!
//
// This option is not named just "web" because of the future desire to merge
// ourbigbook.json directly into options, and we would like a web sub-Object
// which would conflict with this boolean.
options.webMode = false
}
if (!('x_external_prefix' in options)) {
// Used in web to offset the relative paths of issues and editor preview, e.g.
// go/issues/1/username/article
options.x_external_prefix = '';
}
if (!('x_leading_at_to_web' in options)) {
// If \x href starts with @ as in \x[@username] link to OBB Web
// https://ourbigbook.com/username instead of treating it as a regular ID.
options.x_leading_at_to_web = true
}
if (!('x_remove_leading_at' in options)) {
// If true, make \x[@username/someid] link to username/someid without the leading @.
// This is used in Web, where our URLs don't really have the @ sign on them,
// but the username IDs do have the @ sign implicitly added to them.
options.x_remove_leading_at = false;
}
// Internalish options that may get modified by sub-includes/OurBigBookExample in order
// to forward state back up. Maybe we should move them to a subdict to make this clearer
// (moving to extra_returns feels bad because they are also input), but lazy.
//
// Non-indexed-ids: auto-generated numeric ID's like p-1, p-2, etc.
// It is not possible to link to them from inside the document, since links
// break across versions.
if (!('include_path_set' in options)) { options.include_path_set = new Set(); }
if (options.non_indexed_ids === undefined) {
options.non_indexed_ids = {};
}
if (options.indexed_ids === undefined) {
options.indexed_ids = {};
}
if (options.header_tree_stack === undefined) {
options.header_tree_stack = new Map();
}
if (options.header_tree_id_stack === undefined) {
options.header_tree_id_stack = new Map();
}
if (options.is_first_global_header === undefined) {
options.is_first_global_header = true;
}
if (options.refs_to_h === undefined) {
options.refs_to_h = [];
}
if (options.refs_to_x === undefined) {
options.refs_to_x = [];
}
if (options.include_hrefs === undefined) {
options.include_hrefs = {};
}
// Set options from ourbigbook_json. Eventually we want to set every single
// ourbigbook_json option into options. For now let's whitelist them one by one.
options.unsafeXss = resolveOption(options, 'unsafeXss')
if (
( optionsOrig.h === undefined || optionsOrig.h.numbered === undefined ) &&
( options.ourbigbook_json.h !== undefined && options.ourbigbook_json.h.numbered !== undefined)
) {
options.h.numbered = options.ourbigbook_json.h.numbered
}
if (
options.ourbigbook_json.generateSitemap &&
!options.ourbigbook_json.publishRootUrl
) {
throw new Error(`generateSitemap requires publishRootUrl to be set`)
}
return options
}
exports.convertInitOptions = convertInitOptions
function convertInitContext(options={}, extra_returns={}) {
options = convertInitOptions(options)
// Handle scope and IDs that are based on the input path:
//
// - toplevel_has_scope
//
// Set for index files in subdirectories. Is equivalent to
// adding a {scope} to the toplevel header.
//
// - toplevel_parent_scope
//
// Set for files in subdirectories. Means that the (faked non existent)
// parent toplevel header has {scope} set.
//
// - toplevel_id
//
// If true, force the toplevel header to have this ID.
// Otherwise, derive the ID from the title.
// https://docs.ourbigbook.com#the-id-of-the-first-header-is-derived-from-the-filename
//
// TODO hard setting this option here is bad, maybe we should put it in context instead.
options.toplevel_id = undefined;
let root_relpath_shift
let input_dir, basename
let isToplevelIndex = false
if (options.input_path !== undefined) {
;[input_dir, basename] = pathSplit(options.input_path, options.path_sep)
const [basename_noext, ext] = pathSplitext(basename)
if (INDEX_FILE_BASENAMES_NOEXT.has(basename_noext)) {
if (input_dir === '') {
// https://docs.ourbigbook.com#the-toplevel-index-file
options.toplevel_id = undefined;
} else {
// https://docs.ourbigbook.com#the-id-of-the-first-header-is-derived-from-the-filename
options.toplevel_id = input_dir;
options.toplevel_has_scope = true
root_relpath_shift = input_dir
}
isToplevelIndex = input_dir === options.ref_prefix
options.isindex = true
} else {
const [input_path_noext, ext] = pathSplitext(options.input_path)
options.toplevel_id = input_path_noext;
options.isindex = false
}
if (input_dir === '') {
options.toplevel_parent_scope = undefined;
} else {
options.toplevel_parent_scope = input_dir
}
} else {
input_dir = ''
}
if (root_relpath_shift === undefined) {
root_relpath_shift = ''
}
extra_returns.debug_perf = {};
extra_returns.errors = [];
extra_returns.rendered_outputs = {};
const context = {
// Plaintexts for which we want to check if topic IDs exists
automaticTopicLinkPlaintexts: [],
// Topic Ids of interest that we have checked do exist
automaticTopicLinkIds: new Set(),
katex_macros: { ...options.katex_macros },
in_split_headers: false,
in_parse: false,
errors: [],
extra_returns,
forceHeaderHasToc: false,
katex_macros: { ...options.katex_macros },
html_escape: true,
html_is_attr: false,
html_is_href: false,
id_conversion: false,
ignore_errors: false,
/* If true, it means that we are inside a argument that will render as a link.
* Therefore, children must not render as links because nested links are not allowed in HTML.
* One major use case is to have a link inside a header, possibly with {file}:
* = http://example.com
* {file}
*/
in_a: false,
in_header: false,
in_literal: false,
in_parse: false,
in_split_headers: false,
include_path_set: new Set(options.include_path_set),
input_dir,
isToplevelIndex,
last_render: undefined,
macros: macroListToMacros(options.add_test_instrumentation),
options,
// refs_to[false][to_id][type]{from_id: { defined_at: Set[defined_at], child_index: Number }
// refs_to[to][from_id][type]{to_id: { defined_at: Set[defined_at], child_index: Number }
refs_to: {
false: {},
true: {},
},
// Shifts in local \a links due to either:
// - scope + split headers e.g. scope/notindex.html
// - subdirectories
root_relpath_shift,
perf_prev: 0,
// List[String]
// This HTML is added before the next header is rendered, or at the end of conversion after
// the tailing toc if there are no header following. It is then automatically cleared.
renderBeforeNextHeader: [],
// Inside \H we must render X as its href when content is not explicitly given.
// in order to simplify things and not require fetching a graph of IDs.
// But it works quite well with {magic}
renderXAsHref: false,
skipOutputEntry: false,
// Set of all the headers that have synonym set on them.
// Originally used to generate redirects to the heade they point to.
synonym_headers: new Set(),
toc_was_rendered: false,
toplevel_id: options.toplevel_id,
// Map from each toplevel_id to a list of AstNodes with that toplevel_id.
// Updated during render if it is a map, ignored if it is undefined.
toplevel_ids: undefined,
// Output path for the current rendering.
// Gets modified by split headers to point to the split header output path of each split header.
toplevel_output_path: options.outfile,
// undefined: follow defaults set otherwise
// true: force to split headers, e.g. split links
// false: force to nosplit headers, e.g. nosplit links
to_split_headers: undefined,
webUrl: `https://${options.ourbigbook_json.web.host}/`,
}
perfPrint(context, 'start_convert')
return context;
}
exports.convertInitContext = convertInitContext
function convertInitOurbigbookJson(ourbigbook_json={}) {
const ourbigbook_json_orig = ourbigbook_json
ourbigbook_json = lodash.merge({}, OURBIGBOOK_JSON_DEFAULT, ourbigbook_json)
{
if (!('media-providers' in ourbigbook_json)) { ourbigbook_json['media-providers'] = {}; }
{
const media_providers = ourbigbook_json['media-providers'];
for (const media_provider_type of MEDIA_PROVIDER_TYPES) {
if (!(media_provider_type in media_providers)) {
media_providers[media_provider_type] = {};
}
const media_provider = media_providers[media_provider_type];
if (!('title-from-src' in media_provider)) {
media_provider['title-from-src'] = false;
}
}
if (media_providers.local && !('path' in media_providers.local)) {
media_providers.local.path = '';
}
if (media_providers.github && !('remote' in media_providers.github)) {
media_providers.github.remote = 'TODO determine from git remote origin if any';
}
for (const media_provider_name in media_providers) {
const media_provider = media_providers[media_provider_name];
if (!('title-from-src' in media_provider)) {
media_provider['title-from-src'] = false;
}
}
}
{
const web = ourbigbook_json.web
const web_orig = ourbigbook_json_orig.web
if (web_orig !== undefined && !('hostCapitalized' in web_orig) && ('host' in web_orig)) {
web.hostCapitalized = web_orig.host
}
if (web.linkFromStaticHeaderMetaToWeb && web.username === undefined) {
throw new Error(`web.username must be given when web.linkFromStaticHeaderMetaToWeb = true"`)
}
}
}
return ourbigbook_json
}
/** Similar to convertXHref, used for external callers that don't have the context.
* TODO: possibly broken, was returning empty at some point. */
function renderAstFromOpts(astJson, options) {
const context = convertInitContext(options);
context.db_provider = options.db_provider;
return AstNode.fromJSON(astJson, context).render(context)
}
exports.renderAstFromOpts = renderAstFromOpts
/* Like xHref, but called with options as convert,
* so that we don't have to fake a complex context. */
function convertXHref(target_id, options) {
const context = convertInitContext(options)
context.db_provider = options.db_provider
const target_ast = context.db_provider.get(target_id, context)
if (target_ast === undefined) {
return undefined
} else {
return xHref(target_ast, context)
}
}
exports.convertXHref = convertXHref
function dirname(str, sep) {
return pathSplit(str, sep)[0]
}
function duplicateIdErrorMessage(id, path, line, column) {
let message = `duplicate ID "${id}", previous one defined at `;
if (path !== undefined) {
message += `file "${path}" `;
}
message += `line ${line} column ${column}`;
return message
}
exports.duplicateIdErrorMessage = duplicateIdErrorMessage
/** Error message to be rendered inside the generated output itself.
*
* If context is given, escape the message correctly for this context.
*
* @return {String}
*/
function errorMessageInOutput(msg, context) {
let escaped_msg;
if (context === undefined) {
escaped_msg = msg;
} else {
escaped_msg = htmlEscapeContext(context, msg);
}
return `[OURBIGBOOK_ERROR: ${escaped_msg}]`
}
// To overcome wrong pluralize plurals.
// https://github.com/plurals/pluralize/pull/209
function forceLastWordReplace(text, forceLastWord) {
if (forceLastWord) {
const words = text.split(PLURALIZE_WORD_SPLIT_REGEX)
const lastWord = words[words.length - 1]
const forceLastWordCap = []
for (let i = 0; i < forceLastWord.length; i++) {
const forceLastWordC = forceLastWord[i]
if (i < lastWord.length) {
const titleC = lastWord[i]
if (titleC === titleC.toLowerCase()) {
forceLastWordCap.push(forceLastWordC.toLowerCase())
} else {
forceLastWordCap.push(forceLastWordC.toUpperCase())
}
} else {
forceLastWordCap.push(forceLastWordC)
}
}
text = text.slice(0, -lastWord.length) + forceLastWordCap.join('')
}
return text
}
// https://stackoverflow.com/questions/9461621/format-a-number-as-2-5k-if-a-thousand-or-more-otherwise-900
const FORMAT_NUMBER_APPROX_MAP = [
{ value: 1, symbol: "" },
{ value: 1E3, symbol: "k" },
{ value: 1E6, symbol: "M" },
{ value: 1E9, symbol: "G" },
{ value: 1E12, symbol: "T" },
{ value: 1E15, symbol: "P" },
{ value: 1E18, symbol: "E" }
];
function formatNumberApprox(num, digits) {
if (digits === undefined) {
digits = 0;
}
const rx = /\.0+$|(\.[0-9]*[1-9])0+$/;
let i;
for (i = FORMAT_NUMBER_APPROX_MAP.length - 1; i > 0; i--) {
if (num >= FORMAT_NUMBER_APPROX_MAP[i].value) {
break;
}
}
return (num / FORMAT_NUMBER_APPROX_MAP[i].value).toFixed(
digits).replace(rx, "$1") + FORMAT_NUMBER_APPROX_MAP[i].symbol;
}
exports.formatNumberApprox = formatNumberApprox
// Get all possible IDs due to walking up scope resolution
// into the given ids array.
//
// This very slightly duplicates the resolution code in DbProvider.get,
// but it was not trivial to factor them out, so just going for this now.
function getAllPossibleScopeResolutions(current_scope, id, context) {
let ids = []
if (isAbsoluteXref(id, context)) {
ids.push(resolveAbsoluteXref(id, context))
} else {
if (current_scope !== undefined) {
current_scope += Macro.HEADER_SCOPE_SEPARATOR
for (let i = current_scope.length - 1; i > 0; i--) {
if (current_scope[i] === Macro.HEADER_SCOPE_SEPARATOR) {
let pref
if (id === '') {
// Happens on empty link to home on web which has prefix: <>.
pref = current_scope.substring(0, i)
} else {
pref = current_scope.substring(0, i + Macro.HEADER_SCOPE_SEPARATOR.length)
}
ids.push(pref + id)
}
}
}
ids.push(id)
}
return ids
}
function getDescendantCount(tree_node) {
return [
tree_node.descendant_count,
tree_node.word_count,
tree_node.word_count + tree_node.descendant_word_count
];
}
function getDescendantCountHtml(context, tree_node, options) {
if (!context.options.show_descendant_count) {
return undefined
}
if (options.long_style === undefined) {
options.long_style = false;
}
if (options.show_descendant_count === undefined) {
options.show_descendant_count = true;
}
const [descendant_count, word_count, descendant_word_count] = getDescendantCount(tree_node);
let ret;
let word_count_html = ''
function addEntry(cls, long_style, longName, add_test_instrumentation, clsInstr, count) {
// Ideally this would go in a before/after content, but we can't because:
// - we already have one before for the icon
// - can't have multiple befores with different styles: https://stackoverflow.com/questions/11998593/can-i-have-multiple-before-pseudo-elements-for-the-same-element
// - can't be an after because the count comes after
const longStr = long_style ? `${capitalizeFirstLetter(longName)}: ` : ''
// Word Count Recursive.
word_count_html += `<span class="${cls}"> ${longStr}`
if (add_test_instrumentation) {
word_count_html += `<span class="${clsInstr}">`
}
word_count_html += `${formatNumberApprox(count)}`
if (add_test_instrumentation) {
word_count_html += '</span>'
}
word_count_html += '</span>'
}
if (descendant_word_count > 0 && (context.options.add_test_instrumentation || options.show_descendant_count)) {
addEntry('wcntr', options.long_style, 'words', context.options.add_test_instrumentation, 'word-count-descendant', descendant_word_count)
}
if (tree_node.word_count > 0 && context.options.add_test_instrumentation || !options.show_descendant_count) {
addEntry('wcnt', options.long_style, 'words', context.options.add_test_instrumentation, 'word-count', word_count)
}
if (descendant_count > 0 && (context.options.add_test_instrumentation || options.show_descendant_count)) {
addEntry('dcnt', options.long_style, 'articles', context.options.add_test_instrumentation, 'descendant-count', descendant_count)
}
if (word_count_html !== '') {
ret = `<span class="metrics">${word_count_html}</span>`;
}
return ret;
}
/**
*
* @return - inputDirectory: input directory with prefix _file removed if present
*/
function checkAndUpdateLocalLink({
context,
hrefNoEscape,
external,
media_provider_type,
source_location,
}) {
const was_protocol_given = protocolIsGiven(hrefNoEscape)
let inputDirectory
const input_path = context.options.input_path
if (input_path !== undefined) {
inputDirectory = dirname(
input_path,
context.options.path_sep
)
} else {
inputDirectory = '.'
}
const is_absolute = hrefNoEscape[0] === URL_SEP
const is_external = (external !== undefined && external) || (
external === undefined && was_protocol_given
)
// Check existence.
let error = ''
if (!is_external) {
if (hrefNoEscape.length !== 0) {
let check_path;
if (is_absolute) {
check_path = hrefNoEscape.slice(1)
} else {
check_path = path.join(inputDirectory, hrefNoEscape)
}
if (
context.options.fs_exists_sync &&
!context.options.fs_exists_sync(check_path)
) {
error = `link to inexistent local file: ${hrefNoEscape}`;
renderError(context, error, source_location);
error = errorMessageInOutput(error, context)
} else {
const readFileRet = context.options.read_file(check_path, context)
if (
// Fails on web before we implement files on web.
readFileRet !== undefined
) {
const { type } = context.options.read_file(check_path, context)
if (type === 'directory') {
if (context.options.htmlXExtension) {
hrefNoEscape = path.join(hrefNoEscape, 'index.html')
}
}
// Modify external paths to account for scope + --split-headers
let pref = context.root_relpath_shift
if (media_provider_type === 'local') {
if (type === 'directory') {
pref = path.join(pref, DIR_PREFIX)
} else {
pref = path.join(pref, RAW_PREFIX)
}
}
if (!is_absolute) {
pref = path.join(pref, inputDirectory)
}
hrefNoEscape = path.join(pref, hrefNoEscape)
}
}
}
}
return { href: htmlEscapeHrefAttr(hrefNoEscape), error, inputDirectory }
}
// Get description and other closely related attributes.
function getDescription(description_arg, context) {
let description = renderArg(description_arg, context);
let force_separator
if (description === '') {
force_separator = false
} else {
force_separator = true;
}
let multiline_caption
if (description_arg) {
for (const ast of description_arg) {
if (!(
context.macros[ast.macro_name].inline ||
ast.node_type === AstType.PLAINTEXT
)) {
multiline_caption = true
break
}
}
}
if (multiline_caption === undefined) {
multiline_caption = false
}
return { description, force_separator, multiline_caption }
}
function getLinkHtml({
ast,
attrs,
content,
context,
external,
hrefNoEscape,
source_location,
extraReturns,
}) {
if (extraReturns === undefined) {
extraReturns = {}
}
if (!context.in_a) {
if (attrs === undefined) {
attrs = ''
}
Object.assign(extraReturns, checkAndUpdateLocalLink({
context,
external,
hrefNoEscape,
// The only one available for now. One day we could add: \a[some/path]{provider=github}
media_provider_type: 'local',
source_location,
}))
let { href, error } = extraReturns
let testData
if (ast) {
testData = getTestData(ast, context)
} else {
testData = ''
}
return `<a${htmlAttr('href', href)}${attrs}${testData}>${content}</a>${error}`;
} else {
// Don't create a link if we are a child of another link, as that is invalid HTML.
return content
}
}
/** Get the AST from the parent argument of headers or includes. */
function getParentArgumentAst(ast, context, include_options) {
let parent_id;
let parent_ast;
parent_id = magicTitleToId(convertIdArg(ast.args.parent, context), context);
if (isAbsoluteXref(parent_id, context)) {
parent_ast = context.db_provider.get_noscope(resolveAbsoluteXref(parent_id, context), context);
} else {
// We can't use context.db_provider.get here because we don't know who
// the parent node is, because scope can affect that choice.
// https://docs.ourbigbook.com#id-based-header-levels-and-scope-resolution
let sorted_keys = [...include_options.header_tree_stack.keys()].sort((a, b) => a - b);
let largest_level = sorted_keys[sorted_keys.length - 1];
for (let level = largest_level; level > 0; level--) {
let ast = include_options.header_tree_stack.get(level).ast;
if (idIsSuffix(parent_id, ast.id)) {
parent_ast = ast;
break;
}
}
}
return [parent_id, parent_ast];
}
function getRootRelpath(output_path, context) {
// TODO htmlEmbed was split into embedIncludes and embedResources.
// This was likely meant to be embedIncludes, but I don't have a filing test if this is commented out
// so not sure.
const [output_path_dir, output_path_basename] =
pathSplit(output_path, context.options.path_sep);
let root_relpath = path.relative(output_path_dir, '.')
if (root_relpath !== '') {
root_relpath += URL_SEP;
}
return root_relpath
}
function getTestData(ast, context) {
let test_data_arg = ast.args[Macro.TEST_DATA_ARGUMENT_NAME]
if (test_data_arg === undefined) {
return ''
} else {
return htmlAttr(Macro.TEST_DATA_HTML_PROP, renderArg(test_data_arg, context))
}
}
function getTitleAndDescription({
description,
inner,
innerNoDiv,
source,
title,
}) {
let sep
if (innerNoDiv === undefined || isPunctuation(innerNoDiv[innerNoDiv.length - 1])) {
sep = ''
} else {
sep = '.'
}
if (source === undefined) {
source = ''
}
if (source && inner !== undefined) {
source = ' ' + source
}
if (inner !== undefined || source !== '') {
description = ' ' + description
}
return `${title}${sep}${source}${description}`
}
function githubProviderPrefix(context) {
const github = context.options.ourbigbook_json['media-providers'].github
if (github) {
return `https://raw.githubusercontent.com/${github.remote}/master`;
}
}
function checkHasToc(context) {
let root_node = context.header_tree;
if (root_node.children.length === 1) {
root_node = root_node.children[0];
}
return root_node.children.length > 0
}
function hasShorthandLinkEndChars(s) {
for (const c of s) {
if (SHORTHAND_LINK_END_CHARS.has(c)) {
return true
}
}
return false
}
// Ensure that all children and tag targets exist. This is for error checking only.
// https://docs.ourbigbook.com#h-child-argment
function headerCheckChildTagExists(ast, context, childrenOrTags, type) {
let ret = ''
for (let child of childrenOrTags) {
const target_id = magicTitleArgToId(child.args.content, context)
const target_ast = context.db_provider.get(target_id, context, ast.header_tree_node.ast.scope)
if (target_ast === undefined) {
let message = `unknown ${type} id: "${target_id}"`
renderError(context, message, child.source_location)
ret += errorMessageInOutput(message, context)
}
}
return ret
}
/** Make ancestor links for HTML heaader breadcrumb metadata line.
*
* @param {{href: string, content: string}[]} entries
* @return {string}
* */
function htmlAncestorLinks(entries, nAncestors, opts={}) {
let { addTestInstrumentation } = opts
const ret = []
let i = 0
if (nAncestors > ANCESTORS_MAX) {
ret.push(
`<a ${htmlAttr('href', `#${ANCESTORS_ID}`)}` +
`${addTestInstrumentation ? htmlAttr(Macro.TEST_DATA_HTML_PROP, i.toString()) : ''}` +
`> ...</a>`
)
i++
}
for (const entry of entries) {
ret.push(`<a${entry.href}` +
`${addTestInstrumentation ? htmlAttr(Macro.TEST_DATA_HTML_PROP, i.toString()) : ''}` +
`>${i > 0 ? ' ' : ''}${entry.content}</a>`
)
i++
}
return ret.join('')
}
exports.htmlAncestorLinks = htmlAncestorLinks
/** Convert a key value already fully HTML escaped strings
* to an HTML attribute. The callers MUST escape any untrusted chars.
* e.g. with htmlAttrValue.
*
* @param {String} key
* @param {AstArgument} arg
* @return {String} - of form ' a="b"' (with a leading space)
*/
function htmlAttr(key, value) {
return ` ${key}="${value}"`;
}
/** Convert an argument to an HTML attribute value.
*
* @param {AstArgument} arg
* @param {Object} context
* @return {String}
*/
function htmlAttrValue(arg, context) {
return renderArg(arg, cloneAndSet(context, 'html_is_attr', true));
}
function htmlClassAttr(classes) {
return htmlAttr('class', classes.join(' '))
}
function htmlClassesAttr(classes) {
if (classes.length) {
return ` ${htmlClassAttr(classes)}`
} else {
return ''
}
}
function htmlCode(content, attrs) {
return htmlElem('pre', htmlElem('code', content), attrs);
}
/** Helper to convert multiple parameters directly to HTML attributes.
*
* The ID is automatically included.
*
* @param {AstNode} ast
* @param {Object} options
* @param {Array[String]} arg_names - which argument names should be added as properties.
* Only arguments that were given in the text input are used.
* @param {Object[String, AstNode]} custom_args - attributes that were not just passed in
* directly from the input text, but may rather have been calculated from the node.
*/
function htmlRenderAttrs(
ast, context, arg_names=[], custom_args={}
) {
// Determine the arguments.
let args = [];
for (const arg_name in custom_args) {
args.push([arg_name, custom_args[arg_name]]);
}
for (const arg_name of arg_names) {
if (arg_name in ast.args) {
args.push([arg_name, ast.args[arg_name]]);
}
}
// Build the output string.
let ret = '';
for (const name_arg_pair of args) {
const [arg_name, arg] = name_arg_pair;
ret += htmlAttr(arg_name, htmlAttrValue(arg, context));
}
return ret;
}
/**
* Same interface as htmlRenderAttrs, but automatically add the ID to the list
* of arguments.
*/
function htmlRenderAttrsId(
ast, context, arg_names=[], custom_args={}
) {
let id = ast.id;
if (id) {
custom_args[Macro.ID_ARGUMENT_NAME] = [
new PlaintextAstNode(
removeToplevelScope(id, context.toplevel_ast, context),
ast.source_location,
),
];
}
return htmlRenderAttrs(ast, context, arg_names, custom_args);
}
/** Helper for the most common HTML function type that does "nothing magic":
* only has "id" as a possible attribute, and uses ast.args.content as the
* main element child.
*/
function htmlRenderSimpleElem(elem_name, options={}) {
if (!('attrs' in options)) {
options.attrs = {};
}
if (!('wrap' in options)) {
options.wrap = false;
}
if (!('wrap' in options)) {
options.wrapAttrs = {};
}
return function(ast, context) {
let attrs = htmlRenderAttrsId(ast, context);
let extra_attrs_string = '';
for (const key in options.attrs) {
extra_attrs_string += htmlAttr(key, options.attrs[key]);
}
let content = renderArg(ast.args.content, context);
let res = ''
const show_caption = ast.index_id || (ast.validation_output.description && ast.validation_output.description.given)
let elem_attrs
if (show_caption) {
const { description, force_separator, multiline_caption } = getDescription(ast.args.description, context)
const { full: title, inner, innerNoDiv } = xTextBase(ast, context, {
addTitleDiv: true,
href_prefix: htmlSelfLink(ast, context),
force_separator
})
const title_and_description = getTitleAndDescription({ title, description, inner, innerNoDiv })
res += `<div${attrs}><div${multiline_caption ? ` class="${MULTILINE_CAPTION_CLASS}"` : ''}>`;
res += `<div class="caption">${title_and_description}</div>`;
elem_attrs = ''
} else {
elem_attrs = `${extra_attrs_string}${attrs}`
}
res += `<${elem_name}${elem_attrs}${getTestData(ast, context)}>${content}</${elem_name}>`;
if (show_caption) {
res += `</div></div>`;
}
if (options.wrap) {
res = htmlElem('div', res, options.wrapAttrs);
}
return res;
};
}
function htmlElem(tag, content, attrs) {
let ret = '<' + tag;
for (const attr_id in attrs) {
ret += ' ' + attr_id + '="' + htmlEscapeAttr(attrs[attr_id]) + '"'
}
return ret + '>' + content + '</' + tag + '>';
}
function htmlEscapeAttr(str) {
return htmlEscapeContent(str)
.replace(/"/g, '"')
.replace(/'/g, ''')
;
}
exports.htmlEscapeAttr = htmlEscapeAttr
/* This does some extra percent encoding conversions that are not
* stricly required by HTML itself in general, e.g.:
* * '[' -> '%5B'
* * 'á' -> '%C3%A1'
*
* The only thing that we intentionally don't escape is the percent '%' -> '%25'
* itself, as the user might be wanting to do their own explicit encodings.
*
* Notably, 2024 browsers still give you percent encoded strings for Unicode URLs,
* so it would be natural for users to copy and paste that as is.
*
* Most browsers just don't care about any of this and correctly convert for us,
* but let's try to satisfy the validator: https://validator.w3.org/#validate_by_input
*
* Related: https://docs.ourbigbook.com/#link-percent-encoding
*/
function htmlEscapeHrefAttr(str) {
let ret = ''
for (const c of str) {
if (c === '%') {
ret += c
} else {
ret += encodeURI(c)
}
}
return htmlEscapeAttr(ret);
}
function htmlEscapeContent(str) {
return str
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
;
}
exports.htmlEscapeContent = htmlEscapeContent
/** Escape string depending on the current context. */
function htmlEscapeContext(context, str) {
if (context.html_escape) {
if (context.html_is_href) {
return htmlEscapeHrefAttr(str)
} else if (context.html_is_attr) {
return htmlEscapeAttr(str);
} else {
return htmlEscapeContent(str);
}
} else {
return str;
}
}
/** Image handling common to both inline and block images. */
function htmlImg({
// string, e.g. `alt="something"`, escaped
alt,
ast,
context,
external,
inline,
media_provider_type,
rendered_attrs,
relpath_prefix,
// string, unescaped
src,
}) {
let error
;({
error,
href: src,
} = checkAndUpdateLocalLink({
context,
external,
hrefNoEscape: src,
media_provider_type,
source_location: ast.args.src.source_location,
}))
const classes = []
if (ast.validation_output.border.boolean) {
classes.push('border')
}
if (inline) {
classes.push('inline')
}
if (relpath_prefix !== undefined) {
src = path.join(relpath_prefix, src)
}
let cls
if (inline) {
cls = ' class="inline"'
} else {
cls = ''
}
const href = ast.validation_output.link.given ? renderArg(ast.args.link, cloneAndSet(context, 'html_is_href', true)) : src
let html = `<img${htmlAttr('src', src)}${htmlAttr('loading', 'lazy')}${rendered_attrs}${alt}${htmlClassesAttr(classes)}>`
if (!context.in_a) {
html = `<a${htmlAttr('href', href)}>${html}</a>`
}
if (!inline) {
html = `<div class="float-wrap">${html}</div>`
}
return {
html: `${html}${error}`,
src,
};
}
function htmlIsWhitespaceTextNode(ast) {
return ast.node_type === AstType.PLAINTEXT && htmlIsWhitespace(ast.text);
}
// https://stackoverflow.com/questions/2161337/can-we-use-any-other-tag-inside-ul-along-with-li/60885802#60885802
function htmlIsWhitespace(string) {
for (const c of string) {
if (!HTML_ASCII_WHITESPACE.has(c))
return false;
}
return true;
}
function htmlKatexConvert(ast, context) {
let katex_in = renderArg(ast.args.content, cloneAndSet(context, 'html_escape', false))
if (!katex_in.endsWith('\n')) {
katex_in += '\n'
}
try {
return katex.renderToString(
katex_in,
{
globalGroup: true,
macros: context.katex_macros,
// The default is to also add MathML output for blind people.
// However, it adds it with absolute positioning for some reason.
// And as a result, if you add a math formula to the bottom of the editor,
// it generates a toplevel scrollbar on Chromium 84 but not Firefox 79.
output: 'html',
strict: 'error',
throwOnError: true,
}
);
} catch(error) {
// TODO remove the crap KaTeX adds to the end of the string.
// It uses Unicode char hacks to add underlines... and there are two trailing
// chars after the final newline, so the error message is taking up two lines
let message = error.toString().replace(/\n\xcc\xb2$/, '');
renderError(context, message, ast.args.content.source_location);
return errorMessageInOutput(message, context);
}
}
function htmlSelfLink(ast, context) {
return xHrefAttr(
ast,
cloneAndSet(context, 'to_split_headers', context.in_split_headers)
);
}
function htmlTitleAndDescription(ast, context, opts={}) {
let title_and_description = ``
let { description, force_separator, multiline_caption } = getDescription(ast.args.description, context)
let href
if (ast.id) {
href = htmlSelfLink(ast, context)
} else {
href = ''
}
if (ast.index_id || ast.validation_output.description.given) {
const { full: title, inner, innerNoDiv } = xTextBase(ast, context, {
addTitleDiv: opts.addTitleDiv,
force_separator,
href_prefix: href,
})
title_and_description += `<div class="caption">${getTitleAndDescription({ title, description, inner, innerNoDiv })}</div>`
}
return { title_and_description, multiline_caption, href }
}
function idConvertSimpleElem(argname) {
if (argname === undefined) {
argname = Macro.CONTENT_ARGUMENT_NAME
}
return function(ast, context) {
let ret = renderArg(ast.args[argname], context);
let newline = ''
if (!context.macros[ast.macro_name].inline) {
newline = '\n'
}
if (dolog && newline) {
console.log(`idConvertSimpleElem: newline=${require('util').inspect(newline)} ast.macro_name=${ast.macro_name}`)
}
return ret + newline;
};
}
/** bb, aaa/bbb -> false
* bbb, aaa/bbb -> true
* aaa/bbb, aaa/bbb -> true
*/
function idIsSuffix(suffix, full) {
let full_len = full.length;
let suffix_len = suffix.length;
return (
full.endsWith(suffix) &&
(
full_len == suffix_len ||
(
full_len > suffix_len &&
full[full_len - suffix_len - 1] === Macro.HEADER_SCOPE_SEPARATOR
)
)
);
}
// @return [href: string, content: string], both XSS safe.
function linkGetHrefAndContent(ast, context, opts={}) {
let { removeProtocol } = opts
if (removeProtocol === undefined) {
removeProtocol = true
}
const href = renderArg(ast.args.href, cloneAndSet(context, 'html_is_href', true))
const hrefNoEscape = renderArgNoescape(ast.args.href, context)
let content = renderArg(ast.args.content, context)
if (content === '') {
content = renderArg(ast.args.href, context)
if (
removeProtocol &&
!context.id_conversion
) {
content = content.replace(/^https?:\/\//, '')
}
}
return [href, content, hrefNoEscape];
}
// If in split header mode, link to the nosplit version.
// If in the nosplit mode, link to the split version.
function linkToSplitOpposite(ast, context) {
if (context.options.ourbigbook_json.toSplitHeaders) {
return undefined
} else {
let content
let title
let class_
if (context.in_split_headers) {
class_ = 'nosplit'
} else {
class_ = 'split'
}
let other_context = cloneAndSet(context, 'to_split_headers', !context.in_split_headers);
let other_href = xHrefAttr(ast, other_context);
if (
// I'm not going to lie, I bruteforced this. Sue me.
context.options.ourbigbook_json.h.splitDefaultNotToplevel &&
(
context.options.ourbigbook_json.h.splitDefault ||
!context.in_split_headers
)
) {
// This is dirty. But I am dirty.
// But seriously, checking this more cleanly would require
// unpacking a bunch of stuff down below from the toplevel scope removal.
// Related: https://github.com/ourbigbook/ourbigbook/issues/271
const other_href_same = xHrefAttr(ast, context);
if (other_href === other_href_same) {
return undefined
}
}
return `<a${htmlAttr('class', class_)}${other_href}></a>`;
}
}
/**
* @return {Object} dict of macro name to macro
*/
function macroListToMacros(addTestInstrumentation) {
const macros = {};
for (const macro of macroList(addTestInstrumentation)) {
for (const format_id in OUTPUT_FORMATS) {
const convert_func = OUTPUT_FORMATS[format_id].convert_funcs[macro.name]
if (convert_func === undefined) {
throw new Error(`undefined convert function for format "${format_id}" macro "${macro.name}"`)
}
macro.add_convert_function(format_id, convert_func, macro.name);
}
macros[macro.name] = macro;
}
return macros;
}
/** At some point we will generalize this to on-the-fly macro definitions. */
function macroList(addTestInstrumentation) {
return [
...DEFAULT_MACRO_LIST,
...(
addTestInstrumentation
? [
new Macro(
Macro.TEST_SANE_ONLY,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
mandatory: false,
}),
new MacroArgument({
name: 'preferLiteralPositional',
count_words: true,
ourbigbook_output_prefer_literal: true,
mandatory: false,
}),
],
{
named_args: [
new MacroArgument({
name: 'named',
count_words: true,
}),
new MacroArgument({
name: 'preferLiteral',
count_words: true,
ourbigbook_output_prefer_literal: true,
}),
],
}
),
new Macro(
decapitalizeFirstLetter(Macro.TEST_SANE_ONLY),
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
mandatory: true,
}),
],
{
inline: true,
named_args: [
new MacroArgument({
name: 'preferLiteral',
count_words: true,
ourbigbook_output_prefer_literal: true,
}),
],
}
),
]
: []
)
]
}
exports.macroList = macroList
const OURBIGBOOK_EXT = 'bigb';
exports.OURBIGBOOK_EXT = OURBIGBOOK_EXT;
const MEDIA_PROVIDER_TYPES = new Set([
'github',
'local',
'unknown',
'wikimedia',
'youtube',
]);
const media_provider_type_wikimedia_re = new RegExp('^https?://upload.wikimedia.org/wikipedia/commons/');
const media_provider_type_youtube_re = new RegExp('^https?://(www\.)?(youtube.com|youtu.be)/');
const macro_image_video_block_convert_function_wikimedia_source_url = 'https://commons.wikimedia.org/wiki/File:';
const macro_image_video_block_convert_function_wikimedia_source_image_re = new RegExp('^\\d+px-');
const macro_image_video_block_convert_function_wikimedia_source_video_re = new RegExp('^([^.]+\.[^.]+).*');
function macroImageVideoBlockConvertFunction(ast, context) {
let rendered_attrs = htmlRenderAttrs(ast, context, ['height', 'width']);
let figure_attrs = htmlRenderAttrsId(ast, context);
let { description, force_separator, multiline_caption } = getDescription(ast.args.description, context)
let ret = `<div class="figure"><figure${figure_attrs}${multiline_caption ? ` class="${MULTILINE_CAPTION_CLASS}"` : ''}>`
let href_prefix;
if (ast.id !== undefined) {
href_prefix = htmlSelfLink(ast, context);
} else {
href_prefix = undefined;
}
let {
error_message,
media_provider_type,
relpath_prefix,
source,
src,
is_url
} = macroImageVideoResolveParamsWithSource(ast, context);
if (error_message !== undefined) {
return error_message;
}
if (source !== '') {
force_separator = true;
source = `<a${htmlAttr('href', source)}>Source</a>.`;
}
let alt_val;
const has_caption = (ast.id !== undefined) && captionNumberVisible(ast, context);
if (ast.args.alt === undefined) {
if (has_caption) {
alt_val = undefined;
} else {
alt_val = src;
}
} else {
alt_val = renderArgNoescape(ast.args.alt, context)
}
let alt;
if (alt_val === undefined) {
alt = '';
} else {
alt = htmlAttr('alt', htmlEscapeAttr(alt_val));
}
ret += context.macros[ast.macro_name].options.image_video_content_func({
alt,
ast,
context,
is_url,
media_provider_type,
relpath_prefix,
rendered_attrs,
src,
});
if (has_caption) {
const { full: title, inner, innerNoDiv } = xTextBase(ast, context, { addTitleDiv: true, href_prefix, force_separator })
const title_and_description = getTitleAndDescription({ title, description, source, inner, innerNoDiv })
ret += `<figcaption>${title_and_description}</figcaption>`;
}
ret += '</figure></div>';
return ret;
}
/** Convert args such as tag= or \x[]{magic} to their final target ID. */
function magicTitleArgToId(arg, context) {
const id_context = cloneAndSet(context, 'id_conversion', true)
return magicTitleToId(renderArgNoescape(arg, id_context), id_context)
}
/**
* @param string target_id
*/
function magicTitleToId(target_id, context) {
if (target_id.startsWith(Macro.FILE_ID_PREFIX))
return target_id
let ret = titleToIdContext(target_id, { keep_scope_sep: true, magic: true }, context)
if (target_id[0] === AT_MENTION_CHAR) {
ret = AT_MENTION_CHAR + ret
}
return ret
}
// https://stackoverflow.com/questions/44447847/enums-in-javascript-with-es6/49709701#49709701
function makeEnum(arr) {
let obj = {};
for (let val of arr){
obj[val] = Symbol(val);
}
return Object.freeze(obj);
}
function noext(str) {
return str.substring(0, str.lastIndexOf('.'));
}
// https://stackoverflow.com/questions/17781472/how-to-get-a-subset-of-a-javascript-objects-properties/17781518#17781518
function objectSubset(source_object, keys) {
const new_object = {};
keys.forEach((obj, key) => { new_object[key] = source_object[key]; });
return new_object;
}
/* Calculate the output path given a billion parameters.
*
* This is the centerpiece of output path calculation. It is notably used in both:
* - calculating the output paths for a given input
* - calculating where \x links should point
* Since the exact same function is used for both, this guarantees that \x links always
* point to the correct file.
*
* We keep only simple types as inputs to this function (e.g. strings, and notably no AstNode and context)
* to this function so that it can be unit tested, or at least to make it clearer to us what the exact input is.
*
* Countless hours have been wasted writing and debugging this function. It is extremelly hard.
*
* Some of the things this function considers include:
* * split header stuff
**/
function outputPathBase(args={}) {
let {
ast_undefined,
ast_id,
ast_input_path,
ast_is_first_header_in_input_file,
ast_split_default,
ast_toplevel_id,
context_to_split_headers,
ast_input_path_toplevel_id,
path_sep,
splitDefaultNotToplevel,
split_suffix,
toSplitHeadersOverride,
} = args
if (ast_input_path === undefined) {
return undefined
}
const [dirname, basename] = pathSplit(ast_input_path, path_sep);
let renamed_basename_noext = noext(basename)
if (ast_input_path.split(path_sep)[0] !== FILE_PREFIX) {
renamed_basename_noext = renameBasename(renamed_basename_noext)
}
// We are the first header, or something that comes before it.
if (ast_undefined) {
const [dirname_ret, basename_ret] = indexPathFromDirname(dirname, renamed_basename_noext, path_sep)
return { dirname: dirname_ret, basename: basename_ret };
}
// Calculate the base basename_ret and dirname_ret.
let dirname_ret;
let basename_ret;
const to_split_headers = isToSplitHeadersBase(ast_split_default, context_to_split_headers, toSplitHeadersOverride);
if (
ast_is_first_header_in_input_file ||
(
!to_split_headers &&
ast_input_path_toplevel_id === ast_toplevel_id
)
) {
// For toplevel elements in split header mode, we have
// to take care of index and -split suffix.
if (renamed_basename_noext === INDEX_BASENAME_NOEXT) {
// basename_ret
if (
to_split_headers === ast_split_default ||
splitDefaultNotToplevel
) {
// The name is just index.html.
basename_ret = renamed_basename_noext;
} else {
// The name is split.html or nosplit.html.
basename_ret = '';
}
} else {
basename_ret = renamed_basename_noext;
}
dirname_ret = dirname;
} else {
if (to_split_headers && ast_id !== undefined) {
// Non-toplevel elements in split header mode are simple,
// the ID just gives the output path directly.
;[dirname_ret, basename_ret] = pathSplit(ast_id, URL_SEP);
} else {
if (dirname_ret === undefined) {
if (ast_toplevel_id !== undefined) {
;[dirname_ret, basename_ret] = pathSplit(ast_toplevel_id, URL_SEP);
if (basename_ret === '') {
// Happens on home article.
basename_ret = INDEX_BASENAME_NOEXT
}
} else {
;[dirname_ret, basename_ret] = [dirname, renamed_basename_noext]
}
}
}
}
;[dirname_ret, basename_ret] = indexPathFromDirname(dirname_ret, basename_ret, path_sep)
// Add -split, -nosplit or custom suffixes to basename_ret.
let suffix_to_add;
let suffix_added;
if (split_suffix === undefined || split_suffix === '') {
suffix_to_add = to_split_headers ? SPLIT_MARKER_TEXT : NOSPLIT_MARKER_TEXT;
} else {
suffix_to_add = split_suffix;
}
if (
!splitDefaultNotToplevel &&
(
(
to_split_headers &&
(
// To split.html
(
ast_id === ast_toplevel_id &&
!ast_split_default
) ||
// User explcitly gave {splitSuffix}
split_suffix !== undefined
)
) ||
// User gave {splitDefault}, so we link to nosplit.
(
!to_split_headers &&
ast_split_default
)
) &&
!toSplitHeadersOverride
) {
if (basename_ret !== '') {
basename_ret += '-';
}
basename_ret += suffix_to_add;
suffix_added = suffix_to_add;
}
return { dirname: dirname_ret, basename: basename_ret, split_suffix: suffix_added };
}
exports.outputPathBase = outputPathBase;
/** Parse tokens into the AST tree.
*
* @param {Array[Token]} tokens
* @return {Object} extra_returns
* - {Array[ErrorMessage]} errors
* - {Object} ids
* @return {AstNode}
*/
async function parse(tokens, options, context, extra_returns={}) {
perfPrint(context, 'parse_start')
context.in_parse = true
extra_returns.errors = [];
let state = {
extra_returns,
i: 0,
macros: context.macros,
options,
token: tokens[0],
tokens,
};
// Get toplevel arguments such as {title=}, see https://docs.ourbigbook.com#toplevel
const ast_toplevel_args = parseArgumentList(
state, Macro.TOPLEVEL_MACRO_NAME, AstType.MACRO);
if (Macro.CONTENT_ARGUMENT_NAME in ast_toplevel_args) {
parseError(state, `the toplevel arguments cannot contain an explicit content argument`, new SourceLocation(1, 1));
}
// Inject a maybe paragraph token after those arguments.
const paragraph_token = new Token(TokenType.PARAGRAPH, state.token.source_location);
tokens.splice(state.i, 0, paragraph_token);
state.token = paragraph_token;
// Parse the main part of the document as the content
// argument toplevel argument.
const ast_toplevel_content_arg = parseArgument(
state, state.token.source_location);
// Create the toplevel macro itself.
const ast_toplevel = new AstNode(
AstType.MACRO,
Macro.TOPLEVEL_MACRO_NAME,
Object.assign(ast_toplevel_args, {[Macro.CONTENT_ARGUMENT_NAME]: ast_toplevel_content_arg}),
new SourceLocation(1, 1),
);
if (state.token.type !== TokenType.INPUT_END) {
parseError(state, `unexpected tokens at the end of input`);
}
if (context.options.log['ast-pp-simple']) {
console.error('ast-pp-simple: after parse');
console.error(ast_toplevel.toString());
console.error();
}
// Ast post process pass 1
//
// Post process the AST depth first minimally to support includes.
//
// This could in theory be done in a single pass with the next one,
// but that is much more hard to implement and maintain, because we
// have to stich togetegher internal structures to maintain the header
// tree across the includer and included documents.
//
// Another possibility would be to do it in the middle of the initial parse,
// but let's not complicate that further either, shall we?
context.headers_with_include = [];
context.header_tree = new HeaderTreeNode();
perfPrint(context, 'post_process_1')
let prev_non_synonym_header_ast;
let cur_header_level;
let first_header_level;
let first_header;
let header_tree_last_level;
let toplevel_parent_arg = new AstArgument([], new SourceLocation(1, 1));
let todo_visit = [[toplevel_parent_arg, ast_toplevel]];
// IDs that are indexed: you can link to those.
const line_to_id_array = [];
context.line_to_id = function(line) {
let index = binarySearch(line_to_id_array,
[line + 1, undefined], binarySearchLineToIdArrayFn);
if (index < 0) {
index = -(index + 1)
}
index -= 1
if (index === line_to_id_array.length) {
if (line_to_id_array.length > 0) {
index = line_to_id_array.length - 1;
} else {
return undefined;
}
}
return removeToplevelScope(line_to_id_array[index][1], context.toplevel_ast, context);
};
let macro_count_global = 0
const macro_counts = {};
const macro_counts_visible = {};
const headers_from_include = {}
let cur_header_tree_node;
let is_first_header = true;
extra_returns.ids = options.indexed_ids;
let local_db_provider = new DictDbProvider(
options.indexed_ids,
context.refs_to,
);
let db_provider;
if (options.db_provider !== undefined) {
db_provider = new ChainedDbProvider(
local_db_provider,
options.db_provider
);
} else {
db_provider = local_db_provider;
}
context.db_provider = db_provider;
options.include_path_set.add(options.input_path);
const title_ast_ancestors = []
const header_title_ast_ancestors = []
const header_ids = []
let prevAst, ast, parent_arg
let isFirstAst = true
const lintStartsWithH1Header =
context.options.ourbigbook_json.lint.startsWithH1Header &&
!options.fromOurBigBookExample
while (todo_visit.length > 0) {
const pop = todo_visit.pop();
if (pop === 'pop_title_ast_ancestors') {
title_ast_ancestors.pop()
continue
}
if (pop === 'pop_header_title_ast_ancestors') {
header_title_ast_ancestors.pop()
continue
}
if (ast && ast.node_type === AstType.MACRO) {
prevAst = ast
}
;([parent_arg, ast] = pop)
if (parent_arg.argument_name === Macro.TITLE_ARGUMENT_NAME) {
const parent_ast = parent_arg.parent_ast
title_ast_ancestors.push(parent_ast)
todo_visit.push('pop_title_ast_ancestors')
if (parent_ast.macro_name === Macro.HEADER_MACRO_NAME) {
header_title_ast_ancestors.push(parent_ast)
todo_visit.push('pop_header_title_ast_ancestors')
}
}
let parent_arg_push_after = []
let parent_arg_push_before = []
const macro_name = ast.macro_name
const nodeType = ast.node_type
ast.from_include = options.from_include;
ast.from_ourbigbook_example = options.from_ourbigbook_example;
ast.source_location.path = options.input_path;
if (
prevAst &&
prevAst.macro_name === Macro.INCLUDE_MACRO_NAME &&
nodeType !== AstType.PARAGRAPH &&
nodeType !== AstType.NEWLINE &&
macro_name !== Macro.INCLUDE_MACRO_NAME &&
macro_name !== Macro.HEADER_MACRO_NAME
) {
parseError(
state,
`\\Include cannot be in the middle of sections, only at the end of sections immediately before the next header. ` +
`This include was followed by a "${macro_name}" macro`,
prevAst.source_location
);
}
if (
nodeType === AstType.NEWLINE &&
prevAst &&
prevAst.macro_name === Macro.INCLUDE_MACRO_NAME
) {
// Ignore newlines between Includes.
} else if (macro_name === Macro.INCLUDE_MACRO_NAME) {
if (options.forbid_include) {
const error_ast = new PlaintextAstNode(options.forbid_include, ast.source_location);
error_ast.parent_ast = ast.parent_ast;
parent_arg.push(error_ast);
parseError(state, options.forbid_include, ast.source_location);
} else {
const href = renderArgNoescape(ast.args.href, context);
let input_dir, input_basename
if (options.input_path) {
;[input_dir, input_basename] = pathSplit(options.input_path, options.path_sep)
} else {
input_dir = '.'
}
// \Include parent argument handling.
let parent_ast;
let parent_id;
validateAst(ast, context);
if (ast.validation_output.parent.given) {
[parent_id, parent_ast] = getParentArgumentAst(ast, context, options)
if (parent_ast === undefined) {
const message = `${ESCAPE_CHAR}${Macro.INCLUDE_MACRO_NAME} ${HEADER_PARENT_ERROR_MESSAGE}"${parent_id}"`
const error_ast = new PlaintextAstNode(' ' + errorMessageInOutput(message), ast.source_location);
error_ast.parent_ast = ast.parent_ast;
parent_arg.push(error_ast);
parseError(state, message, ast.args.parent.source_location);
}
}
if (parent_ast === undefined) {
parent_ast = options.cur_header;
}
let parent_ast_header_level
let parent_ast_header_tree_node
let include_id = href
let curScope
if (parent_ast) {
curScope = parent_ast.scope
} else if (options.cur_header && options.cur_header.scope) {
curScope = options.cur_header.scope
}
if (curScope) {
include_id = curScope + Macro.HEADER_SCOPE_SEPARATOR + include_id
}
if (include_id === TOPLEVEL_INDEX_ID) {
const message = `cannot ${ESCAPE_CHAR}${Macro.INCLUDE_MACRO_NAME} the toplevel index file https://docs.ourbigbook.com/#the-toplevel-index-file`
const error_ast = new PlaintextAstNode(' ' + errorMessageInOutput(message), ast.source_location);
error_ast.parent_ast = ast.parent_ast;
parent_arg.push(error_ast);
parseError(state, message, ast.source_location);
} else {
if (parent_ast === undefined) {
parent_ast_header_level = 0
parent_ast_header_tree_node = context.header_tree
} else {
// Possible on include without a parent header.
parent_ast_header_tree_node = parent_ast.header_tree_node;
parent_ast_header_level = parent_ast_header_tree_node.get_level();
addToRefsTo(
include_id,
context,
parent_ast.id,
REFS_TABLE_PARENT,
{
child_index: parent_ast_header_tree_node.children.length,
source_location: ast.source_location,
},
);
parent_ast.includes.push(href);
}
const peek_ast = todo_visit[todo_visit.length - 1][1];
if (peek_ast.node_type === AstType.PLAINTEXT && peek_ast.text === '\n') {
todo_visit.pop();
}
// https://github.com/ourbigbook/ourbigbook/issues/215
const read_include_ret = await (options.read_include(href, input_dir));
if (read_include_ret === undefined) {
if (
// On the local filesystem, this doesn't matter.
// But on the server it does, as we don't know about the other includes
// before they are processed.
context.options.render
) {
let message = `could not find include: "${href}"`;
parseError(
state,
message,
ast.source_location,
);
parent_arg.push(new PlaintextAstNode(message, ast.source_location));
}
} else {
const [include_path, include_content] = read_include_ret;
if (options.include_path_set.has(include_path)) {
let message = `circular include detected to: "${include_path}"`;
parseError(
state,
message,
ast.source_location,
);
parent_arg.push(new PlaintextAstNode(message, ast.source_location));
} else {
let new_child_nodes;
if (options.embed_includes) {
new_child_nodes = await parseInclude(
include_content,
options,
parent_ast_header_level,
include_path,
href,
{
errors: extra_returns.errors,
}
);
options.include_path_set.add(include_path);
} else {
const from_include = true
// Don't merge into a single file, render as a dummy header and an xref link instead.
const header_ast = new AstNode(
AstType.MACRO,
Macro.HEADER_MACRO_NAME,
{
'level': new AstArgument(
[
new PlaintextAstNode(
(parent_ast_header_level + 1).toString(),
)
],
),
[Macro.TITLE_ARGUMENT_NAME]: new AstArgument( [
// Will be patched in later in order to group all DB queries at the end of parse,
// as this requires getting an ID from DB.
new PlaintextAstNode('TODO patchme')
],
),
},
undefined,
{
force_no_index: true,
from_include,
id: include_id,
level: parent_ast_header_level + 1,
},
);
options.include_hrefs[include_id] = header_ast
headers_from_include[include_id] = header_ast
if (options.cur_header !== undefined) {
header_ast.scope = options.cur_header.scope
}
header_ast.header_tree_node = new HeaderTreeNode(header_ast, parent_ast_header_tree_node);
parent_ast_header_tree_node.add_child(header_ast.header_tree_node);
new_child_nodes = [
header_ast,
new AstNode(
AstType.PARAGRAPH,
),
new AstNode(
AstType.MACRO,
Macro.PARAGRAPH_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
[
new AstNode(
AstType.MACRO,
Macro.X_MACRO_NAME,
{
'href': new AstArgument(
[
new PlaintextAstNode(href)
],
),
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
[
new PlaintextAstNode(
'This section is present in another page, follow this link to view it.',
)
],
),
},
undefined,
),
],
),
},
undefined,
),
new AstNode(AstType.PARAGRAPH),
];
for (const child_node of new_child_nodes) {
child_node.set_source_location(ast.source_location)
child_node.set_recursively({
count_words: false,
from_include,
})
}
}
if (options.output_format === OUTPUT_FORMAT_OURBIGBOOK) {
if (options.render_include) {
parent_arg.push(ast)
}
} else {
// Push all included nodes, but don't recurse because:
// - all child includes will be resolved on the sub-render call
// - the current header level must not move, so that consecutive \Include
// calls won't nest into one another
for (const new_child_node of new_child_nodes) {
new_child_node.parent_ast = ast.parent_ast;
}
parent_arg.push(...new_child_nodes);
}
}
}
}
}
} else if (macro_name === Macro.OURBIGBOOK_EXAMPLE_MACRO_NAME) {
const ourbigbookExampleOptions = cloneAndSet(options, 'fromOurBigBookExample', true)
if (ourbigbookExampleOptions.output_format === OUTPUT_FORMAT_OURBIGBOOK) {
parent_arg.push(ast)
}
// We need to add these asts on OUTPUT_FORMAT_OURBIGBOOK so that we can extract their IDs later on.
// We then just don't render them at render time.
const new_asts = [
new AstNode(
AstType.MACRO,
Macro.CODE_MACRO_NAME.toUpperCase(),
{ [Macro.CONTENT_ARGUMENT_NAME]: ast.args.content },
ast.source_location,
),
new AstNode(
AstType.MACRO,
Macro.PARAGRAPH_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
[
new PlaintextAstNode(
'which renders as:',
ast.source_location,
)
],
ast.source_location
),
},
ast.source_location,
),
new AstNode(
AstType.MACRO,
Macro.QUOTE_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: await parseInclude(
renderArgNoescape(ast.args.content, cloneAndSet(context, 'id_conversion', true)),
ourbigbookExampleOptions,
0,
ourbigbookExampleOptions.input_path,
undefined,
{
start_line: ast.source_location.line + 1,
errors: extra_returns.errors,
}
),
},
ast.source_location,
),
]
for (const new_ast of new_asts) {
new_ast.set_recursively({
from_ourbigbook_example: true,
})
}
parent_arg.push(...new_asts);
} else {
// Not OurBigBookExample.
if (macro_name === Macro.HEADER_MACRO_NAME) {
if (is_first_header) {
ast.id = context.toplevel_id
if (options.toplevel_has_scope) {
// We also need to fake an argument here, because that will
// get serialized to the database, which is needed for
// toplevel scope removal from external links.
const scope_arg = new AstArgument([], ast.source_location);
ast.add_argument('scope', scope_arg);
}
if (options.toplevel_parent_scope !== undefined) {
ast.scope = options.toplevel_parent_scope;
}
ast.is_first_header_in_input_file = true;
}
ast.subdir = convertIdArg(ast.args.subdir, context)
// Required by calculateId.
validateAst(ast, context);
if (
!is_first_header &&
!ast.isSynonym() &&
options.forbid_multiheader
) {
parseError(
state,
options.forbid_multiheader,
ast.source_location,
);
parent_arg.push(new PlaintextAstNode(options.forbid_multiheader, ast.source_location));
}
let is_synonym = ast.isSynonym()
const header_level = ast.validation_output.level.positive_nonzero_integer
// splitDefault propagation to children.
if (ast.validation_output.splitDefault.given) {
ast.split_default = ast.validation_output.splitDefault.boolean;
} else if (options.cur_header !== undefined) {
ast.split_default = options.cur_header.split_default || options.cur_header.split_default_children;
} else {
if (options.ourbigbook_json.h.splitDefaultNotToplevel) {
ast.split_default = false
ast.split_default_children = options.ourbigbook_json.h.splitDefault
} else {
ast.split_default = options.ourbigbook_json.h.splitDefault
}
}
if (is_synonym) {
if (options.cur_header === undefined) {
const message = `the first header of an input file cannot be a synonym`;
parseError(state, message, ast.args.synonym.source_location);
// Hack it to behave like a non-synonym. This is the easiest way to avoid further errors.
is_synonym = false
} else {
if (header_level !== 1) {
const message = `synonym headers must be h1, got: ${header_level}`;
parseError(state, message, ast.args.level.source_location);
}
ast.synonym = options.cur_header.id;
if (ast.args[Macro.TITLE2_ARGUMENT_NAME] !== undefined) {
if (ast.args[Macro.TITLE2_ARGUMENT_NAME].asts.length > 1) {
parseError(state, `synonym headers can have at most one ${Macro.TITLE2_ARGUMENT_NAME} argument`, ast.args[Macro.TITLE2_ARGUMENT_NAME].source_location);
}
if (ast.args[Macro.TITLE2_ARGUMENT_NAME].asts[0].args[Macro.CONTENT_ARGUMENT_NAME].asts.length > 0) {
parseError(state, `the ${Macro.TITLE2_ARGUMENT_NAME} of synonym headers must be empty`, ast.args[Macro.TITLE2_ARGUMENT_NAME].source_location);
}
options.cur_header.title2s.push(ast);
}
context.synonym_headers.add(ast);
}
}
if (!is_synonym) {
prev_non_synonym_header_ast = ast;
options.cur_header = ast;
cur_header_level = header_level + options.h_parse_level_offset;
}
let parent_tree_node_error = false;
let parent_id;
if (ast.validation_output.parent.given) {
let parent_ast;
if (is_synonym) {
const message = `synonym and parent are incompatible`;
parseError(state, message, ast.args.level.source_location);
}
if (header_level !== 1) {
const message = `header with parent argument must have level equal 1`;
ast.args[Macro.TITLE_ARGUMENT_NAME].push(
new PlaintextAstNode(' ' + errorMessageInOutput(message), ast.source_location)
);
parseError(state, message, ast.args.level.source_location);
}
;[parent_id, parent_ast] = getParentArgumentAst(ast, context, options);
let parent_tree_node;
if (parent_ast !== undefined) {
parent_tree_node = options.header_tree_id_stack.get(parent_ast.id);
}
if (parent_tree_node === undefined) {
parent_tree_node_error = true;
} else {
cur_header_level = parent_tree_node.get_level() + 1;
}
}
if ('level' in ast.args) {
// Hack the level argument of the final AST to match for consistency.
ast.args.level = new AstArgument([
new PlaintextAstNode(cur_header_level.toString(), ast.args.level.source_location)],
ast.args.level.source_location);
}
// lint['h-parent']
if (
context.options.ourbigbook_json.lint['h-parent'] &&
!ast.isSynonym()
) {
let message;
if (
context.options.ourbigbook_json.lint['h-parent'] === 'parent' &&
!ast.validation_output.parent.given &&
!is_first_header
) {
message = `no parent given with lint['h-parent'] = "parent"`;
} else if (
context.options.ourbigbook_json.lint['h-parent'] === 'number' &&
ast.validation_output.parent.given
) {
message = `parent given with lint['h-parent'] = "number"`;
}
if (message) {
parseError(state, message, ast.source_location);
parent_arg_push_after.push(new PlaintextAstNode(errorMessageInOutput(message), ast.source_location));
}
}
// lint['h-tag']
if (
context.options.ourbigbook_json.lint['h-tag'] !== undefined
) {
let message;
let arg;
if (
context.options.ourbigbook_json.lint['h-tag'] === 'child' &&
ast.validation_output[Macro.HEADER_TAG_ARGNAME].given
) {
message = `tag given with lint['h-tag'] = "child"`;
arg = ast.args[Macro.HEADER_TAG_ARGNAME]
} else if (
context.options.ourbigbook_json.lint['h-tag'] === 'tag' &&
ast.validation_output[Macro.HEADER_CHILD_ARGNAME].given
) {
message = `child given with lint['h-tag'] = "tag"`;
arg = ast.args[Macro.HEADER_CHILD_ARGNAME]
}
if (message) {
parseError(state, message, arg.source_location);
parent_arg_push_after.push(new PlaintextAstNode(errorMessageInOutput(message), arg.source_location));
}
}
if (options.is_first_global_header) {
first_header = ast;
first_header_level = cur_header_level;
header_tree_last_level = cur_header_level - 1;
options.header_tree_stack.set(header_tree_last_level, context.header_tree);
options.is_first_global_header = false;
}
let header_level_skip_error;
if (is_synonym) {
ast.scope = options.cur_header.scope;
} else {
cur_header_tree_node = new HeaderTreeNode(ast, options.header_tree_stack.get(cur_header_level - 1));
if (cur_header_level - header_tree_last_level > 1) {
header_level_skip_error = header_tree_last_level;
}
if (cur_header_level < first_header_level) {
parseError(
state,
`header level ${cur_header_level} is smaller than the level of the first header of the document ${first_header_level}`,
ast.args.level.source_location,
);
}
const parent_tree_node = options.header_tree_stack.get(cur_header_level - 1);
if (parent_tree_node !== undefined) {
parent_tree_node.add_child(cur_header_tree_node);
const parent_ast = parent_tree_node.ast;
if (parent_ast !== undefined) {
let scope = parent_ast.calculate_scope();
// The ast might already have a scope here through less common means such as
// being in a subdirectory.
if (ast.scope) {
if (scope === undefined) {
scope = ''
} else {
scope += Macro.HEADER_SCOPE_SEPARATOR
}
scope += ast.scope
}
ast.scope = scope;
}
}
const old_tree_node = options.header_tree_stack.get(cur_header_level);
options.header_tree_stack.set(cur_header_level, cur_header_tree_node);
if (
// Possible on the first insert of a level.
old_tree_node !== undefined &&
// Possible if the level is not an integer.
old_tree_node.ast !== undefined
) {
options.header_tree_id_stack.delete(old_tree_node.ast.id);
}
header_tree_last_level = cur_header_level;
}
ast.header_tree_node = cur_header_tree_node
const isHomeHeader = is_first_header && context.isToplevelIndex
// Must come after the header tree step is mostly done, because scopes influence ID,
// and they also depend on the parent node.
;({ macro_count_global } = calculateId(
ast,
context,
options.non_indexed_ids,
options.indexed_ids,
macro_counts,
macro_count_global,
macro_counts_visible,
state,
line_to_id_array,
{
isHomeHeader,
}
))
// Sanity checks.
if (
cur_header_level === 1 &&
!is_first_header &&
options.forbid_multi_h1 &&
//!ast.synonym
!ast.isSynonym()
) {
const msg = `only one level 1 header is allowed in this conversion, extra header has ID: "${ast.id}"`
parseError(state, msg, ast.source_location);
parent_arg.push(new PlaintextAstNode(msg, ast.source_location));
}
if (isHomeHeader) {
// The main ID is empty ''. And now also create what is written in the header e.g.
// = My toplevel id
// as a virtual synonym of that empty ID. This was originally introduced for Home
// to work, and also to solve https://github.com/ourbigbook/ourbigbook/issues/334
const synonymAst = lodash.clone(ast)
synonymAst.id = undefined
if (Macro.ID_ARGUMENT_NAME in synonymAst.args) {
delete synonymAst.args[Macro.ID_ARGUMENT_NAME]
}
synonymAst.validation_output[Macro.ID_ARGUMENT_NAME].given = false
synonymAst.synonym = context.options.ref_prefix
// This is only normally propagated on post_process_2. But here we have a weird
// out of tree thing that does not show up in post_process_2, so lets set it here
// as we know it for ure for this case. Ugly but works.
synonymAst.toplevel_id = ''
const { idIsEmpty } = calculateId(
synonymAst,
context,
options.non_indexed_ids,
options.indexed_ids,
macro_counts,
macro_count_global,
macro_counts_visible,
state,
line_to_id_array,
{
ignoreEmptyId: true,
isHomeHeader: false,
}
)
if (!idIsEmpty) {
addToRefsTo(
prev_non_synonym_header_ast.id,
context,
synonymAst.id,
REFS_TABLE_SYNONYM,
{ source_location: ast.source_location, }
)
}
}
// Now stuff that must come after calculateId.
header_ids.push(ast.id)
// https://github.com/ourbigbook/ourbigbook/issues/100
if (parent_tree_node_error) {
const message = HEADER_PARENT_ERROR_MESSAGE + parent_id;
ast.args[Macro.TITLE_ARGUMENT_NAME].push(
new PlaintextAstNode(' ' + errorMessageInOutput(message), ast.source_location));
parseError(state, message, ast.args.parent.source_location);
}
if (is_synonym) {
addToRefsTo(
prev_non_synonym_header_ast.id,
context,
ast.id,
REFS_TABLE_SYNONYM,
{ source_location: ast.source_location, }
)
} else {
if (header_level_skip_error !== undefined) {
const message = `skipped a header level from ${header_level_skip_error} to ${cur_header_level}`;
ast.args[Macro.TITLE_ARGUMENT_NAME].push(
new PlaintextAstNode(' ' + errorMessageInOutput(message), ast.source_location));
parseError(state, message, ast.args.level.source_location);
}
options.header_tree_id_stack.set(cur_header_tree_node.ast.id, cur_header_tree_node);
}
if (parent_arg_push_before.length) {
parent_arg_push_before = parent_arg_push_before.concat([new AstNode(
AstType.PARAGRAPH, undefined, undefined, ast.source_location)
])
}
// Push a paragraph separator after the header when adding nodes after.
if (parent_arg_push_after.length) {
parent_arg_push_after = [new AstNode(
AstType.PARAGRAPH, undefined, undefined, ast.source_location
)].concat(parent_arg_push_after)
}
// Add children/tags to the child database.
// https://docs.ourbigbook.com#h-child-argment
for (const [argname, child] of [
[Macro.HEADER_CHILD_ARGNAME, true],
[Macro.HEADER_TAG_ARGNAME, false],
]) {
const tags_or_children = ast.args[argname]
if (tags_or_children !== undefined) {
for (const tag_or_child of tags_or_children) {
const target_id = magicTitleArgToId(tag_or_child.args.content, context)
for (const target_id_with_scope of getAllPossibleScopeResolutions(ast.calculate_scope(), target_id, context)) {
options.refs_to_h.push({
ast,
child,
source_location: tag_or_child.source_location,
target_id: target_id_with_scope,
type: REFS_TABLE_X_CHILD,
})
}
}
}
}
is_first_header = false
} else if (macro_name === Macro.X_MACRO_NAME) {
// refs database updates.
validateAst(ast, context)
let target_id = convertFileIdArg(ast, ast.args.href, context)
if (
// Otherwise the ref would be added to the DB and DB checks would fail.
!ast.validation_output.topic.boolean
) {
const fetch_plural = ast.validation_output.magic.boolean
if (fetch_plural) {
target_id = magicTitleToId(target_id, context)
}
const cur_scope = options.cur_header
? options.cur_header.calculate_scope()
// This came up when converting comments on Web. Another option might be
// to use the input_path instead.
// https://github.com/ourbigbook/ourbigbook/issues/277
: context.options.toplevel_parent_scope
for (const target_id_with_scope of getAllPossibleScopeResolutions(cur_scope, target_id, context)) {
options.refs_to_x.push({
ast,
title_ast_ancestors: Object.assign([], title_ast_ancestors),
target_id: target_id_with_scope,
inflected: false,
})
}
if (fetch_plural) {
// In the case of magic, also fetch a singularized version from DB. We don't know
// which one is the correct one, so just fetch both and decide at render time.
const href_arg = ast.args.href
const last_ast = href_arg.get(href_arg.length() - 1);
if (
// Possible for unterminated shorthand link.
last_ast &&
last_ast.node_type === AstType.PLAINTEXT
) {
const old_text = last_ast.text
const new_text = pluralizeWrap(old_text, 1)
if (new_text !== old_text) {
last_ast.text = new_text
const target_id = magicTitleToId(convertIdArg(ast.args.href, context), context)
last_ast.text = old_text
for (const target_id_with_scope of getAllPossibleScopeResolutions(cur_scope, target_id, context)) {
options.refs_to_x.push({
ast,
title_ast_ancestors: Object.assign([], title_ast_ancestors),
target_id: target_id_with_scope,
inflected: true,
})
}
}
}
}
}
}
// Push this node into the parent argument list.
// This allows us to skip nodes, or push multiple nodes if needed.
parent_arg.push(...parent_arg_push_before);
parent_arg.push(ast);
parent_arg.push(...parent_arg_push_after);
// Recurse.
for (const arg_name in ast.args) {
const arg = ast.args[arg_name];
for (let i = arg.length() - 1; i >= 0; i--) {
todo_visit.push([arg, arg.get(i)]);
}
// We make the new argument be empty so that children can
// decide if they want to push themselves or not.
arg.reset()
}
}
if (
macro_name !== Macro.TOPLEVEL_MACRO_NAME &&
macro_name !== undefined
) {
if (
isFirstAst && lintStartsWithH1Header &&
(
macro_name !== Macro.HEADER_MACRO_NAME ||
ast.validation_output.level.positive_nonzero_integer !== 1
)
) {
parseError(state, `files must start with a header of level 1, found instead ${
macro_name === Macro.HEADER_MACRO_NAME
? `a header of level ${ast.validation_output.level.positive_nonzero_integer}`
: `${ESCAPE_CHAR}${macro_name}`
}`, ast.source_location)
}
isFirstAst = false
if (context.options.h1Only) {
break
}
}
}
if (isFirstAst && lintStartsWithH1Header) {
parseError(state, `files cannot be empty`, new SourceLocation(1, 1))
}
if (context.options.log['ast-pp-simple']) {
console.error('ast-pp-simple: after pass 1');
console.error(ast_toplevel.toString());
console.error();
}
let fetch_header_tree_ids_rows
let fetch_ancestors_rows
const prefetch_file_ids = new Set()
if (!options.from_include) {
// Ast post process pass 2
//
// Post process the AST pre-order depth-first search after
// inclusions are resolved to support things like:
//
// - the shorthand but necessary paragraphs double newline syntax
// - automatic ul parent to li and table to tr
// - remove whitespace only text children from ul
// - extract all IDs into an ID index
//
// Normally only the toplevel includer will enter this code section.
perfPrint(context, 'post_process_2')
// Calculate header_tree_top_level.
//
// - if a header of this level is present in the document,
// there is only one of it. This implies for example that
// it does not get numerical prefixes like "1.2.3 My Header".
// when rendering, and it does not show up in the ToC.
context.header_tree_top_level = first_header_level;
if (context.header_tree.children.length === 1) {
const toplevel_header_node = context.header_tree.children[0];
const toplevel_header_ast = toplevel_header_node.ast;
if (toplevel_header_node.children.length > 0) {
toplevel_header_node.children[0].ast.toc_header = true;
}
context.toplevel_ast = toplevel_header_ast;
} else {
context.toplevel_ast = undefined;
if (context.header_tree.children.length > 0) {
context.header_tree.children[0].ast.toc_header = true;
}
}
// Not modified by split headers.
context.nosplit_toplevel_ast = context.toplevel_ast
let toplevel_parent_arg = new AstArgument([], new SourceLocation(1, 1));
{
const todo_visit = [[toplevel_parent_arg, ast_toplevel]];
while (todo_visit.length > 0) {
let [parent_arg, ast] = todo_visit.pop();
const macro_name = ast.macro_name;
const macro = context.macros[macro_name];
if (
macro_name === Macro.TOPLEVEL_MACRO_NAME &&
ast.parent_ast !== undefined
) {
// Prevent this from happening. When this was committed originally,
// it actually worked and output an `html` inside another `html`.
// Maybe we could do something with iframe, but who cares about that?
const message = `the "${Macro.TOPLEVEL_MACRO_NAME}" cannot be used explicitly`;
ast = new PlaintextAstNode(errorMessageInOutput(message), ast.source_location);
parseError(state, message, ast.source_location);
} else if (
ast.macro_name === Macro.HEADER_MACRO_NAME
) {
propagateNumbered(ast, context)
// Propagate toplevel_id for headers.
if (
// Fails for include dummies. No patience to find a better way now.
ast.validation_output.toplevel &&
ast.validation_output.toplevel.boolean
) {
ast.toplevel_id = ast.id
} else {
const parent_tree_node = ast.header_tree_node.parent_ast
if (
parent_tree_node === undefined ||
parent_tree_node.ast === undefined
) {
if (ast.is_first_header_in_input_file) {
ast.toplevel_id = ast.id
} else {
// Multiple toplevel h1, so after the first one we pick the same toplevel_id as the first one has.
if (parent_tree_node !== undefined) {
ast.toplevel_id = parent_tree_node.children[0].ast.toplevel_id
}
}
} else {
ast.toplevel_id = parent_tree_node.ast.toplevel_id
}
}
}
// Dump index of headers with includes.
if (ast.includes.length > 0) {
context.headers_with_include.push(ast);
}
if (
// This only happens in the following cases
// * \x are validated before for magic stuff
// * when you have include without embed,
// where we do a context.db_provider.get, and we set the attributes to it
// and the thing comes out of serialization validated.
macro_name !== Macro.HEADER_MACRO_NAME && !ast.validated
) {
validateAst(ast, context);
}
// Push this node into the parent argument list.
// This allows us to skip nodes, or push multiple nodes if needed.
parent_arg.push(ast);
// Loop over the child arguments. We do this rather than recurse into them
// to be able to easily remove or add nodes to the tree during this AST
// post-processing.
//
// Here we do sibling-type transformations that need to loop over multiple
// direct children in one go, such as:
//
// - auto add ul to li
// - remove whitespace only text children from ul
for (const arg_name in ast.args) {
// The following passes consecutively update arg.
let arg = ast.args[arg_name];
let macro_arg = macro.name_to_arg[arg_name];
// Handle elide_link_only.
if (
// Possible for error nodes.
macro_arg !== undefined &&
macro_arg.elide_link_only &&
arg.length() === 1 &&
arg.get(0).macro_name === Macro.LINK_MACRO_NAME
) {
const href_arg = arg.get(0).args.href;
href_arg.parent_ast = ast;
arg = href_arg;
}
// Resolve line breaks.
{
const new_arg = AstArgument.copyEmpty(arg)
for (let i = 0; i < arg.length(); i++) {
const child = arg.get(i)
if (
child.node_type === AstType.NEWLINE
) {
const prev = arg.get(i - 1)
const next = arg.get(i + 1)
if (
(prev && prev.node_type !== AstType.PARAGRAPH && context.macros[prev.macro_name].inline) &&
(next && next.node_type !== AstType.PARAGRAPH && context.macros[next.macro_name].inline)
) {
new_arg.push(new AstNode(
AstType.MACRO,
Macro.LINE_BREAK_MACRO_NAME,
{},
child.source_location
))
}
} else {
new_arg.push(child)
}
}
arg = new_arg
}
// Child loop that adds table tr implicit parents to th and td.
// This needs to be done on a separate pass before the tr implicit table adding.
// It is however very similar to the other loop: the only difference is that we eat up
// a trailing paragraph if followed by another.
{
const new_arg = AstArgument.copyEmpty(arg)
for (let i = 0; i < arg.length(); i++) {
let child_node = arg.get(i);
let new_child_nodes = [];
let new_child_nodes_set = false;
if (child_node.node_type === AstType.MACRO) {
const child_macro_name = child_node.macro_name;
if (
child_macro_name == Macro.TD_MACRO_NAME ||
child_macro_name == Macro.TH_MACRO_NAME
) {
const auto_parent_name = Macro.TR_MACRO_NAME;
const auto_parent_name_macro = state.macros[auto_parent_name];
if (
ast.macro_name !== auto_parent_name
) {
const start_auto_child_node = child_node;
const new_arg_auto_parent = AstArgument.copyEmpty(child_node.source_location)
while (i < arg.length()) {
const arg_i = arg.get(i);
if (arg_i.node_type === AstType.MACRO) {
if (
arg_i.macro_name == Macro.TD_MACRO_NAME ||
arg_i.macro_name == Macro.TH_MACRO_NAME
) {
new_arg_auto_parent.push(arg_i);
} else {
break;
}
} else if (arg_i.node_type === AstType.PARAGRAPH) {
if (i + 1 < arg.length()) {
const arg_i_next_macro_name = arg.get(i + 1).macro_name;
if (
arg_i_next_macro_name == Macro.TD_MACRO_NAME ||
arg_i_next_macro_name == Macro.TH_MACRO_NAME
) {
// Ignore this paragraph, it is actually only a separator between two \tr.
i++;
}
}
break;
} else if (
auto_parent_name_macro.name_to_arg[Macro.CONTENT_ARGUMENT_NAME].remove_whitespace_children &&
htmlIsWhitespaceTextNode(arg_i)
) {
// Ignore the whitespace node.
} else {
break;
}
i++;
}
new_child_nodes_set = true;
new_child_nodes = new AstArgument([new AstNode(
AstType.MACRO,
auto_parent_name,
{
[Macro.CONTENT_ARGUMENT_NAME]: new_arg_auto_parent,
},
start_auto_child_node.source_location,
{
parent_ast: child_node.parent_ast
}
)], child_node.source_location);
// Because the for loop will advance past it.
i--;
}
}
}
if (!new_child_nodes_set) {
new_child_nodes = [child_node];
}
new_arg.push(...new_child_nodes);
}
arg = new_arg;
}
// Child loop that adds ul and table implicit parents.
{
const new_arg = AstArgument.copyEmpty(arg)
for (let i = 0; i < arg.length(); i++) {
let child_node = arg.get(i);
let new_child_nodes = [];
let new_child_nodes_set = false;
if (
(arg_name in macro.name_to_arg) &&
macro.name_to_arg[arg_name].remove_whitespace_children &&
htmlIsWhitespaceTextNode(child_node)
) {
new_child_nodes_set = true;
} else if (child_node.node_type === AstType.MACRO) {
let child_macro_name = child_node.macro_name;
let child_macro = state.macros[child_macro_name];
if (child_macro.auto_parent !== undefined) {
// Add ul and table implicit parents.
const auto_parent_name = child_macro.auto_parent;
const auto_parent_name_macro = state.macros[auto_parent_name];
if (
ast.macro_name !== auto_parent_name &&
!child_macro.auto_parent_skip.has(ast.macro_name)
) {
const start_auto_child_node = child_node;
const new_arg_auto_parent = AstArgument.copyEmpty(child_node)
while (i < arg.length()) {
const arg_i = arg.get(i);
if (arg_i.node_type === AstType.MACRO) {
if (state.macros[arg_i.macro_name].auto_parent === auto_parent_name) {
new_arg_auto_parent.push(arg_i);
} else {
break;
}
} else if (
auto_parent_name_macro.name_to_arg[Macro.CONTENT_ARGUMENT_NAME].remove_whitespace_children &&
htmlIsWhitespaceTextNode(arg_i)
) {
// Ignore the whitespace node.
} else {
break;
}
i++;
}
new_child_nodes_set = true;
new_child_nodes = new AstArgument([new AstNode(
AstType.MACRO,
auto_parent_name,
{
[Macro.CONTENT_ARGUMENT_NAME]: new_arg_auto_parent,
},
start_auto_child_node.source_location,
{
parent_ast: child_node.parent_ast
}
)], child_node.source_location);
// Because the for loop will advance past it.
i--;
}
}
}
if (!new_child_nodes_set) {
new_child_nodes = [child_node];
}
new_arg.push(...new_child_nodes);
}
arg = new_arg;
}
// Child loop that adds paragraphs.
{
let paragraph_indexes = [];
for (let i = 0; i < arg.length(); i++) {
const child_node = arg.get(i);
if (child_node.node_type === AstType.PARAGRAPH) {
paragraph_indexes.push(i);
}
}
if (paragraph_indexes.length > 0) {
const new_arg = AstArgument.copyEmpty(arg)
if (paragraph_indexes[0] > 0) {
parseAddParagraph(state, ast, new_arg, arg, 0, paragraph_indexes[0], options);
}
let paragraph_start = paragraph_indexes[0] + 1;
for (let i = 1; i < paragraph_indexes.length; i++) {
const paragraph_index = paragraph_indexes[i];
parseAddParagraph(state, ast, new_arg, arg, paragraph_start, paragraph_index, options);
paragraph_start = paragraph_index + 1;
}
if (paragraph_start < arg.length()) {
parseAddParagraph(state, ast, new_arg, arg, paragraph_start, arg.length(), options);
}
arg = new_arg;
}
}
// Push children to continue the search. We make the new argument be empty
// so that children can decide if they want to push themselves or not.
{
const new_arg = AstArgument.copyEmpty(arg)
const macro_arg_count_words = macro_arg !== undefined && macro_arg.count_words
for (let i = arg.length() - 1; i >= 0; i--) {
const child_ast = arg.get(i)
// Propagate count_words.
if (!child_ast.count_words || !macro_arg_count_words) {
child_ast.count_words = false
child_ast.word_count = 0
}
todo_visit.push([new_arg, child_ast]);
}
// Update the argument.
ast.args[arg_name] = new_arg;
}
}
}
}
if (context.options.log['ast-pp-simple']) {
console.error('ast-pp-simple: after pass 2');
console.error(ast_toplevel.toString());
console.error();
}
// Now do a pass that collects information that may be affected by
// the tree modifications of the previous step, e.g. ID generation.
//
// We also do some error checking here, to account for change that might have been
// made in previous steps.
perfPrint(context, 'post_process_3')
{
const todo_visit = [[ast_toplevel, undefined, []]]
let cur_header_tree_node
while (todo_visit.length > 0) {
const [ast, inlineOnly, ancestorArgs] = todo_visit.pop();
const macro_name = ast.macro_name;
const macro = context.macros[macro_name];
let children_in_header;
if (inlineOnly && !macro.inline) {
parseError(
state,
`${inlineOnly} but macro ${ESCAPE_CHAR}${macro_name} is not inline`,
ast.source_location,
)
}
let fetchPlaintextArgs = true
for (const [ancestorArg, ancestorMacroName] of ancestorArgs) {
if (ancestorArg !== undefined) {
if (
!ancestorArg.automaticTopicLinks ||
!ancestorArg.count_words ||
ancestorArg.renderLinksAsPlaintext
) {
fetchPlaintextArgs = false
}
if (ancestorArg.cannotContain.has(macro_name)) {
const msg = `argument "${ancestorArg.name}" of macro ` +
`${ESCAPE_CHAR}${ancestorMacroName} cannot contain ` +
`the macro ${ESCAPE_CHAR}${macro_name}`
ast.renderAsError = msg
parseError( state, msg, ast.source_location)
}
}
}
if (macro_name === Macro.HEADER_MACRO_NAME) {
// TODO start with the toplevel.
cur_header_tree_node = ast.header_tree_node;
children_in_header = true;
if (ast.parent_ast.macro_name !== Macro.TOPLEVEL_MACRO_NAME) {
parseError(
state,
`headers (\\${Macro.HEADER_MACRO_NAME}) must be directly at document toplevel, not as children of other elements. Parent was instead: ${ESCAPE_CHAR}${ast.parent_ast.macro_name}`,
ast.source_location,
)
}
} else {
if (
context.options.automaticTopicLinksMaxWords &&
macro_name === Macro.PLAINTEXT_MACRO_NAME &&
fetchPlaintextArgs
) {
context.automaticTopicLinkPlaintexts.push(ast.text)
ast.automaticTopicLinks = true
}
ast.header_tree_node = new HeaderTreeNode(ast, cur_header_tree_node);
if (ast.in_header) {
children_in_header = true;
} else {
if (cur_header_tree_node !== undefined) {
cur_header_tree_node.word_count += ast.word_count;
}
children_in_header = false;
}
if (cur_header_tree_node !== undefined) {
ast.scope = cur_header_tree_node.ast.calculate_scope();
} else {
// https://github.com/ourbigbook/ourbigbook/issues/277
ast.scope = context.options.toplevel_parent_scope
}
// Header IDs already previously calculated for parent= so we don't redo it in that case.
let ret = calculateId(ast, context, options.non_indexed_ids, options.indexed_ids, macro_counts,
macro_count_global, macro_counts_visible, state, line_to_id_array);
macro_count_global = ret.macro_count_global
// Propagate some header properties to non-header children.
// This allows us to save some extra DB fetches, at the cost of making the JSON slightly larger.
// and duplicating data a bit. But whaterver, simpler code with less JOINs.
if (ast.header_tree_node.parent_ast !== undefined && ast.header_tree_node.parent_ast.ast !== undefined) {
const header_ast = ast.header_tree_node.parent_ast.ast
ast.split_default = header_ast.split_default;
ast.is_first_header_in_input_file = header_ast.is_first_header_in_input_file;
// TODO calculate a value for this when there is not existing header_ast, e.g.:
//
// ``
// abc
//
// = First header
// ``
//
// currently leaves abc without toplevel_id. This would make it impossible to link from other files to it.
// Not fixing it now because this is forbidden in Web articles.
ast.toplevel_id = header_ast.toplevel_id
}
// Ensure that all rows have the same number of columns.
if (macro_name === Macro.TR_MACRO_NAME) {
const tableAst = ast.parent_ast
if (tableAst.macro_name === Macro.TABLE_MACRO_NAME) {
const firstRow = tableAst.args[Macro.CONTENT_ARGUMENT_NAME].asts[0]
if (firstRow.macro_name === Macro.TR_MACRO_NAME) {
const nCols = ast.args[Macro.CONTENT_ARGUMENT_NAME].asts.length
const firstNCols = firstRow.args[Macro.CONTENT_ARGUMENT_NAME].asts.length
if (nCols !== firstNCols) {
parseError(
state,
`${ESCAPE_CHAR}${Macro.TABLE_MACRO_NAME} row has ${nCols} columns but the first row in table has ${firstNCols}`,
ast.source_location,
)
}
}
}
}
// TH must be the first thing in table.
if (macro_name === Macro.TH_MACRO_NAME) {
const rowAst = ast.parent_ast
if (rowAst.macro_name === Macro.TR_MACRO_NAME) {
const argIdx = rowAst.parent_argument_index
if (argIdx !== 0) {
parseError(
state,
`${ESCAPE_CHAR}${Macro.TH_MACRO_NAME} row can only be the first row of a table was ${argIdx}`,
ast.source_location,
)
}
}
}
}
// Push children to continue the search. We make the new argument be empty
// so that children can decide if they want to push themselves or not.
for (const arg_name in ast.args) {
const arg = ast.args[arg_name];
for (let i = arg.length() - 1; i >= 0; i--) {
let newInlineOnly
const macro_arg = macro.name_to_arg[arg_name]
let newAncestorArgs = [...ancestorArgs, [macro_arg, macro_name]]
if (inlineOnly) {
newInlineOnly = inlineOnly
} else {
if (macro_arg && macro_arg.inlineOnly) {
newInlineOnly = `argument "${arg_name}" of macro ${ESCAPE_CHAR}${macro.name} can only contain inline macros`
} else if (macro.inline) {
newInlineOnly = `${ESCAPE_CHAR}${macro.name} is an inline macro, and inline macros can only contain inline macros`
}
}
const childAst = arg.get(i)
const childMacroName = childAst.macro_name
if (
(
macro_name === Macro.UNORDERED_LIST_MACRO_NAME ||
macro_name === 'Ol'
) &&
arg_name === Macro.CONTENT_ARGUMENT_NAME &&
childMacroName !== Macro.LIST_ITEM_MACRO_NAME
) {
parseError(
state,
`macro \\${macro_name} argument "${Macro.CONTENT_ARGUMENT_NAME}" can only contain \\${Macro.LIST_ITEM_MACRO_NAME}, found instead ${childAst.node_type.toString()} with ${childAst.macro_name}`,
childAst.source_location,
)
}
if (
macro_name === Macro.TABLE_MACRO_NAME &&
arg_name === Macro.CONTENT_ARGUMENT_NAME &&
childMacroName !== Macro.TR_MACRO_NAME
) {
parseError(
state,
`macro \\${macro_name} argument "${Macro.CONTENT_ARGUMENT_NAME}" can only contain \\${Macro.TR_MACRO_NAME}, found instead ${childAst.node_type.toString()} with ${childAst.macro_name}`,
childAst.source_location,
)
}
if (
macro_name === Macro.TR_MACRO_NAME &&
arg_name === Macro.CONTENT_ARGUMENT_NAME &&
childMacroName !== Macro.TD_MACRO_NAME &&
childMacroName !== Macro.TH_MACRO_NAME
) {
parseError(
state,
`macro \\${macro_name} argument "${Macro.CONTENT_ARGUMENT_NAME}" can only contain \\${Macro.TH_MACRO_NAME} or \\${Macro.TD_MACRO_NAME}, found instead ${childAst.node_type.toString()} with ${childAst.macro_name}`,
childAst.source_location,
)
}
childAst.in_header = children_in_header
todo_visit.push([childAst, newInlineOnly, newAncestorArgs])
}
}
}
}
if (context.options.log['ast-pp-simple']) {
console.error('ast-pp-simple: after pass 3');
console.error(ast_toplevel.toString());
console.error();
}
// Now for some final operations that don't go over the entire Ast Tree.
perfPrint(context, 'post_process_4')
// Setup refs DB.
for (const ref of options.refs_to_h) {
const target_id_effective = xChildDbEffectiveId(
ref.target_id,
context,
ref.ast
)
if (ref.child) {
addToRefsTo(target_id_effective, context, ref.ast.id, ref.type, { source_location: ref.source_location });
} else {
addToRefsTo(ref.ast.id, context, target_id_effective, ref.type, { source_location: ref.source_location });
}
}
for (const ref of options.refs_to_x) {
const ast = ref.ast
const target_id_effective = xChildDbEffectiveId(
ref.target_id,
context,
ast
)
const parent_id = ast.get_local_header_parent_id();
if (
// Happens on some special elements e.g. the ToC.
parent_id !== undefined
) {
// TODO add test and enable this possible fix.
if (
// We don't want the "This section is present in another page" to count as a link.
!ast.from_include
) {
// Update xref database for incoming links.
const from_ids = addToRefsTo(
target_id_effective,
context,
parent_id,
REFS_TABLE_X,
{
source_location: ast.source_location,
inflected: ref.inflected,
}
)
}
// Update xref database for child/parent relationships.
{
let toid, fromid;
if (ast.validation_output.child.boolean) {
fromid = parent_id;
toid = target_id_effective;
} else if (ast.validation_output.parent.boolean) {
toid = parent_id;
fromid = target_id_effective;
}
if (toid !== undefined) {
addToRefsTo(toid, context, fromid, REFS_TABLE_X_CHILD, { source_location: ast.source_location });
}
}
}
for (const title_ast of ref.title_ast_ancestors) {
addToRefsTo(
target_id_effective,
context,
title_ast.id,
REFS_TABLE_X_TITLE_TITLE,
{ source_location: ast.source_location }
);
}
}
const first_toplevel_child = ast_toplevel.args.content.get(0);
if (first_toplevel_child !== undefined) {
first_toplevel_child.first_toplevel_child = true;
}
if (context.options.log['ast-pp-simple']) {
console.error('ast-pp-simple: after pass 4');
console.error(ast_toplevel.toString());
console.error();
}
if (context.options.render) {
perfPrint(context, 'db_queries')
if (options.db_provider !== undefined) {
const prefetch_ids = new Set()
for (const ref of options.refs_to_h) {
prefetch_ids.add(ref.target_id)
}
for (const ref of options.refs_to_x) {
// TODO as an easy optimization, just get all refs defined on the current source files.
// These are already resolved for scope and \x magic pluralization, so the final query
// would be much smaller than this.
const id = ref.target_id
prefetch_ids.add(id)
prefetch_file_ids.add(id)
const ast = ref.ast
if (
context.options.output_format === OUTPUT_FORMAT_OURBIGBOOK &&
ast.validation_output.p.boolean
) {
const id_plural = pluralizeWrap(id.replaceAll(ID_SEPARATOR, ' ')).replaceAll(' ', ID_SEPARATOR)
if (id !== id_plural) {
// We need to also fetch this to decide if we can use magic plural or not, in case
// there are separate IDs for both singular and plural. But we don't want to put it into
// refs_to_x, because we don't want it to end up in the Refs database.
prefetch_ids.add(id_plural)
}
}
}
if (options.parent_id) {
prefetch_ids.add(options.parent_id)
// TODO needed?
//prefetch_file_ids.add({ from_id: options.parent_id, to_id: toplevel_ast.id })
// If we managed to fake the ref here (which does not exist on DB yet),
// then we would get the ancestor list working as well.
// But it is not so easy because:
// row.from is stored in row format, not as Ast
}
if (options.forbid_include) {
for (const id of header_ids) {
// Hack the id_cache. This is needed to setup the tree. This is likely something like what is somewhat
// done in the \Include reconciliation below.
options.db_provider.id_cache[id] = context.db_provider.get(id, context)
}
} else {
for (const id in options.include_hrefs) {
// We need the target it to be able to render the dummy include title
// with link to the real content.
prefetch_ids.add(id)
}
}
let automaticTopicLinkIdsToFetch = new Set()
if (options.automaticTopicLinksMaxWords) {
for (const plaintext of context.automaticTopicLinkPlaintexts) {
const plaintextSplitPunct = plaintext.split(AUTOMATIC_TOPIC_LINK_PUNCTUATION_REGEX)
for (const p of plaintextSplitPunct) {
const idComps = titleToIdContext(p, undefined, context).split(ID_SEPARATOR)
for (let i = 0; i < idComps.length; i++) {
for (let j = 1; j <= options.automaticTopicLinksMaxWords; j++) {
const id = idComps.slice(i, i + j).join(ID_SEPARATOR)
if (automaticTopicLinkConsiderId(id)) {
automaticTopicLinkIdsToFetch.add(id)
automaticTopicLinkIdsToFetch.add(pluralizeWrap(id, 1))
}
}
}
}
}
}
// QUERRY EVERYTHING AT ONCE NOW!
let get_noscopes_base_fetch, get_refs_to_fetch, get_refs_to_fetch_reverse, topicIds
;[
get_noscopes_base_fetch,
get_refs_to_fetch,
get_refs_to_fetch_reverse,
fetch_header_tree_ids_rows,
fetch_ancestors_rows,
automaticTopicLinkIds,
] = await Promise.all([
options.db_provider.get_noscopes_base_fetch(
Array.from(prefetch_ids),
new Set(),
context,
),
// TODO merge the following two refs fetch into one single DB query. Lazy now.
// Started prototype at: https://github.com/ourbigbook/ourbigbook/tree/merge-ref-cache
// The annoying part is deciding what needs to go in which direction of the cache.
options.db_provider.get_refs_to_fetch(
[
// These are needed to render each header.
// Shows on parents.
REFS_TABLE_PARENT,
// Shows on tags.
REFS_TABLE_X_CHILD,
// This is needed for the Incoming links at the bottom of each output file.
REFS_TABLE_X,
],
header_ids,
{
context,
ignore_paths_set: context.options.include_path_set,
},
),
// This is needed to generate the "tagged" at the end of each output file.
options.db_provider.get_refs_to_fetch(
[
REFS_TABLE_X_CHILD,
],
header_ids,
{
context,
reversed: true,
ignore_paths_set: context.options.include_path_set,
}
),
context.options.render_metadata
? options.db_provider.fetch_header_tree_ids(
options.forbid_include
// Since \Include is not allowed, we must check every header.
// Parent relations are encoded directly on DB only. Used in Web.
? header_ids
: Object.keys(options.include_hrefs)
)
: []
,
options.db_provider.fetch_ancestors(context.toplevel_id, context),
// topicIds
options.automaticTopicLinksMaxWords
? options.db_provider.fetchTopics(automaticTopicLinkIdsToFetch)
: []
])
context.automaticTopicLinkIds = new Set(automaticTopicLinkIds)
}
// Reconcile the dummy include header with our actual knowledge from the DB, e.g.:
// * patch the ID of the include headers.
// Has to be deferred here after DB fetch obviously because we need he DB data.
// This is hair pulling stuff. There has to be a better way...
for (const href in options.include_hrefs) {
const header_ast = options.include_hrefs[href]
const target_ast = context.db_provider.get(href, context, header_ast.scope);
if (target_ast === undefined) {
let message = `ID in include not found on database: "${href}", ` +
`needed to calculate the internal link title. Did you forget to convert all files beforehand?`;
header_ast.args[Macro.TITLE_ARGUMENT_NAME].get(0).text = errorMessageInOutput(message)
if (options.render) {
parseError(state, message, header_ast.source_location);
}
} else {
if (target_ast.is_first_header_in_input_file) {
// We want the rendered placeholder to use its parent numbered, so as to follow the includer's numbering scheme,
// but the descendants to follow what they would actually render in the output as so they will show correctly on ToC.
header_ast.add_argument('numbered', new AstArgument(
[
new PlaintextAstNode(context.options.h.numbered ? Macro.BOOLEAN_ARGUMENT_TRUE : Macro.BOOLEAN_ARGUMENT_FALSE, header_ast.source_location),
],
header_ast.source_location,
))
}
header_ast.splitDefault = target_ast.splitDefault
propagateNumbered(header_ast, context)
header_ast.set_source_location(target_ast.source_location)
header_ast.header_tree_node.update_ancestor_counts(target_ast.header_tree_node_word_count)
for (const argname in target_ast.args) {
if (
// We have to patch the level of the target ID (1) do our new dummy one in the current tree.
argname !== 'level' &&
argname !== 'wiki' &&
target_ast.validation_output[argname].given
) {
header_ast.args[argname] = target_ast.args[argname]
}
}
}
// This is a bit nasty and duplicates the header processing code,
// but it is a bit hard to factor them out since this is a magic include header,
// and all includes and headers must be parsed concurrently since includes get
// injected under the last header.
validateAst(header_ast, context);
if (target_ast !== undefined) {
// We modify the cache here to ensure that the header ID has the full header_tree_node, which
// then gets feched from \x{full} (notably ToC) in order to show the link number there.
//
// Yes, this erase IDs that come from other Includes, but we don´t have a use case for that
// right now, e.g. the placholder include header does not show parents.
target_ast.header_tree_node = header_ast.header_tree_node
target_ast.header_parent_ids = []
}
}
let build_header_tree_asts
if (options.db_provider !== undefined) {
build_header_tree_asts = context.options.db_provider.build_header_tree(
fetch_header_tree_ids_rows, { context })
context.options.db_provider.fetch_ancestors_build_tree(
fetch_ancestors_rows, context)
}
if (context.options.db_provider !== undefined) {
const prefetch_files = new Set()
// TODO I tried to do this as a JOIN from inside get_refs_to_fetch to replace these fetches,
// but then if I remove these, some tests start to fail. Understand why and remove this one day.
for (const prefetch_file_id of prefetch_file_ids) {
const prefetch_ast = context.db_provider.get_noscope(prefetch_file_id, context)
if (
// Possible in some error cases.
prefetch_ast !== undefined
) {
prefetch_files.add(prefetch_ast.source_location.path)
}
}
if (options.db_provider !== undefined) {
for (const ast of build_header_tree_asts) {
prefetch_files.add(ast.source_location.path)
}
}
if (prefetch_files.size) {
await context.options.db_provider.fetch_files(Array.from(prefetch_files), context)
}
}
for (const id in headers_from_include) {
const ast = headers_from_include[id]
// This to ensure that the ast we get from \x will have a consistent
// numbering with the local parent.
// This code will likely be removed if we do:
// https://github.com/ourbigbook/ourbigbook/issues/188
const cached_ast = context.db_provider.get(ast.id, context)
if (
// Possible in error cases and TODO apparently some non-error too.
cached_ast !== undefined
) {
cached_ast.numbered = ast.numbered
}
}
}
}
context.in_parse = false
perfPrint(context, 'parse_end')
return ast_toplevel;
}
// Maybe add a paragraph after a \n\n.
function parseAddParagraph(
state, ast, new_arg, arg, paragraph_start, paragraph_end, options
) {
parseLogDebug(state, 'function: parseAddParagraph');
parseLogDebug(state, 'paragraph_start: ' + paragraph_start);
parseLogDebug(state, 'paragraph_end: ' + paragraph_end);
parseLogDebug(state);
if (paragraph_start < paragraph_end) {
const macro = state.macros[arg.get(paragraph_start).macro_name];
const slice = arg.slice(paragraph_start, paragraph_end);
if (macro.inline || slice.length() > 1) {
// If the first element after the double newline is inline content,
// create a paragraph and put all elements until the next paragraph inside
// that paragraph.
new_arg.push(
new AstNode(
AstType.MACRO,
Macro.PARAGRAPH_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: slice
},
arg.get(paragraph_start).source_location,
{
parent_ast: ast,
}
)
);
} else {
// Otherwise, don't create the paragraph, and keep all elements as they were.
new_arg.push(...slice);
}
}
}
// Consume one token.
function parseConsume(state) {
state.i += 1;
if (state.i < state.tokens.length) {
state.token = state.tokens[state.i];
} else {
throw new Error('programmer error');
}
parseLogDebug(state, 'function: parseConsume');
parseLogDebug(state, 'state.i = ' + state.i.toString())
parseLogDebug(state, 'state.token = ' + JSON.stringify(state.token));
parseLogDebug(state);
return state.token;
}
function parseLogDebug(state, msg='') {
if (state.options.log.parse) {
console.error('parse: ' + msg);
}
}
// Input: e.g. in `\Image[img.jpg]{height=123}` this parses the `[img.jpg]{height=123}`.
// Return value: dict with arguments.
function parseArgumentList(state, macro_name, macro_type) {
parseLogDebug(state, 'function: parseArgumentList');
parseLogDebug(state, 'state = ' + JSON.stringify(state.token));
parseLogDebug(state);
const args = {};
const macro = state.macros[macro_name];
let name_to_arg;
if (
// Happens in some error cases.
macro !== undefined
) {
name_to_arg = macro.name_to_arg
}
let positional_arg_count = 0;
while (
// End of stream.
state.token.type !== TokenType.INPUT_END &&
(
state.token.type === TokenType.POSITIONAL_ARGUMENT_START ||
state.token.type === TokenType.NAMED_ARGUMENT_START ||
state.token.type === TokenType.NEWLINE
)
) {
let arg_name;
let open_token = state.token;
const nextToken = state.tokens[state.i + 1]
const nextTokenType = nextToken ? nextToken.type : undefined
if (open_token.type === TokenType.NEWLINE) {
if (
nextTokenType === TokenType.POSITIONAL_ARGUMENT_START ||
nextTokenType === TokenType.NAMED_ARGUMENT_START
) {
// Ignore newline between two arguments.
parseConsume(state);
} else {
break
}
} else {
// Consume the *_ARGUMENT_START token out.
parseConsume(state);
if (open_token.type === TokenType.POSITIONAL_ARGUMENT_START) {
if (macro_type === AstType.ERROR) {
arg_name = positional_arg_count.toString();
} else {
if (positional_arg_count >= macro.positional_args.length) {
parseError(state,
`unknown named macro argument "${arg_name}" of macro "${macro_name}"`,
open_token.source_location,
);
arg_name = positional_arg_count.toString();
} else {
arg_name = macro.positional_args[positional_arg_count].name;
}
positional_arg_count += 1;
}
} else {
// Named argument.
arg_name = state.token.value;
if (macro_type !== AstType.ERROR && !(arg_name in macro.named_args)) {
parseError(state,
`unknown named macro argument "${arg_name}" of macro "${macro_name}"`,
state.token.source_location
);
}
// Parse the argument name out.
parseConsume(state);
}
const arg_children = parseArgument(state, open_token.source_location);
if (state.token.type !== closingToken(open_token.type)) {
parseError(state, `unclosed argument "${open_token.value}"`, open_token.source_location);
}
if (
// Happens in some error cases, e.g. \\undefinedMacro[aa]
macro !== undefined
) {
const macro_arg = name_to_arg[arg_name];
const multiple = macro_arg !== undefined && macro_arg.multiple
if (arg_name in args) {
if (!multiple) {
// https://github.com/ourbigbook/ourbigbook/issues/101
parseError(state,
`named argument "${arg_name}" given multiple times`,
open_token.source_location,
)
}
} else {
if (multiple) {
args[arg_name] = new AstArgument([], open_token.source_location)
} else {
args[arg_name] = arg_children;
}
}
if (multiple) {
args[arg_name].push(new AstNode(
AstType.MACRO,
'Comment',
{ [Macro.CONTENT_ARGUMENT_NAME]: arg_children },
open_token.source_location,
))
}
}
if (state.token.type !== TokenType.INPUT_END) {
// Consume the *_ARGUMENT_END token out.
parseConsume(state);
}
}
}
return args;
}
/**
* Input: e.g. in `\Image[img.jpg]{height=123}` this parses the `img.jpg` and the `123`.
* @return AstArgument
*/
function parseArgument(state, open_argument_source_location) {
const arg_children = new AstArgument([], open_argument_source_location);
while (
state.token.type !== TokenType.INPUT_END &&
state.token.type !== TokenType.POSITIONAL_ARGUMENT_END &&
state.token.type !== TokenType.NAMED_ARGUMENT_END
) {
// The recursive case: the argument is a lists of macros, go into all of them.
arg_children.push(parseMacro(state));
}
return arg_children;
}
// Parse one macro. This is the centerpiece of the parsing!
// Input: e.g. in `\Image[img.jpg]{height=123}` this parses the entire string.
function parseMacro(state) {
parseLogDebug(state, 'function: parseMacro');
parseLogDebug(state, 'state = ' + JSON.stringify(state.token));
parseLogDebug(state);
if (state.token.type === TokenType.MACRO_NAME) {
const macro_name = state.token.value;
const macro_source_location = state.token.source_location;
let macro_type;
const unknown_macro_message = `unknown macro name: "${macro_name}"`;
if (macro_name in state.macros) {
macro_type = AstType.MACRO;
} else {
macro_type = AstType.ERROR;
parseError(state, unknown_macro_message);
}
// Consume the MACRO_NAME token out.
parseConsume(state);
const args = parseArgumentList(state, macro_name, macro_type);
if (macro_type === AstType.ERROR) {
return new AstNode(
macro_type,
Macro.PLAINTEXT_MACRO_NAME,
{},
state.token.source_location,
{text: errorMessageInOutput(unknown_macro_message)},
);
} else {
return new AstNode(macro_type, macro_name, args, macro_source_location);
}
} else if (state.token.type === TokenType.PLAINTEXT) {
// Non-recursive case.
let node = new PlaintextAstNode(
state.token.value,
state.token.source_location,
);
// Consume the PLAINTEXT node out.
parseConsume(state);
return node;
} else if (state.token.type === TokenType.PARAGRAPH) {
let node = new AstNode(
AstType.PARAGRAPH,
undefined,
undefined,
state.token.source_location,
);
parseConsume(state);
return node;
} else if (state.token.type === TokenType.NEWLINE) {
let node = new AstNode(
AstType.NEWLINE,
undefined,
undefined,
state.token.source_location,
);
parseConsume(state);
return node;
} else {
let error_message
if (
state.token.type === TokenType.POSITIONAL_ARGUMENT_START ||
state.token.type === TokenType.NAMED_ARGUMENT_START
) {
error_message = `stray open argument character: '${state.token.value}', maybe you want to escape it with '\\'`;
} else {
// Generic error message.
error_message = `unexpected token ${state.token.type.toString()}`;
}
parseError(state, error_message);
let node = new PlaintextAstNode(
errorMessageInOutput(error_message),
state.token.source_location,
);
// Consume past whatever happened to avoid an infinite loop.
parseConsume(state);
return node;
}
state.i += 1;
}
function parseError(state, message, source_location) {
let new_source_location;
if (source_location === undefined) {
new_source_location = new SourceLocation();
} else {
new_source_location = source_location.clone();
}
new_source_location.path = state.options.input_path;
if (new_source_location.line === undefined)
new_source_location.line = state.token.source_location.line;
if (new_source_location.column === undefined)
new_source_location.column = state.token.source_location.column;
state.extra_returns.errors.push(new ErrorMessage(
message, new_source_location));
}
const parseNonNegativeIntegerStrictRe = /^\d+$/
function parseNonNegativeIntegerStrict(s) {
if (!s.match(parseNonNegativeIntegerStrictRe))
return NaN
return parseInt(s)
}
function pathJoin(dirname, basename, sep) {
let ret = dirname;
if (ret !== '') {
ret += sep;
}
return ret + basename;
}
function pathSplit(str, sep) {
const dir_sep_index = str.lastIndexOf(sep)
if (dir_sep_index == -1) {
return ['', str];
} else {
return [str.substring(0, dir_sep_index), str.substring(dir_sep_index + 1)]
}
}
exports.pathSplit = pathSplit
function pathSplitext(str) {
const sep_index = str.lastIndexOf('.')
if (sep_index == -1) {
return [str, ''];
} else {
return [str.substring(0, sep_index), str.substring(sep_index + 1)]
}
}
exports.pathSplitext = pathSplitext
function protocolGet(url) {
const match = /^([a-zA-Z]+):\/\//.exec(url)
if (match) {
return match[0]
} else {
return null
}
}
function protocolIsGiven(url) {
return protocolGet(url) !== null
}
function protocolIsKnown(src) {
for (const known_url_protocol of KNOWN_URL_PROTOCOLS) {
if (src.startsWith(known_url_protocol)) {
return true;
}
}
return false;
}
exports.protocolIsKnown = protocolIsKnown
// https://docs.ourbigbook.com#scope
function removeToplevelScope(id, toplevel_ast, context) {
const fixedScopeRemoval = context.options.fixedScopeRemoval
if (fixedScopeRemoval !== undefined) {
return id.slice(fixedScopeRemoval)
}
if (
toplevel_ast !== undefined &&
id === toplevel_ast.id
) {
if (toplevel_ast.scope !== undefined) {
return id.substring(toplevel_ast.scope.length + 1)
}
return id;
} else {
return id.substring(calculateScopeLength(toplevel_ast))
}
}
// https://docs.ourbigbook.com#index-files
const INDEX_BASENAME_NOEXT = 'index';
exports.INDEX_BASENAME_NOEXT = INDEX_BASENAME_NOEXT;
const INDEX_BASENAME = INDEX_BASENAME_NOEXT + '.' + OURBIGBOOK_EXT
exports.INDEX_BASENAME = INDEX_BASENAME
const INDEX_FILE_BASENAMES_NOEXT = new Set([
INDEX_BASENAME_NOEXT,
]);
exports.INDEX_FILE_BASENAMES_NOEXT = INDEX_FILE_BASENAMES_NOEXT;
const IO_RENAME_MAP = {};
for (let i of INDEX_FILE_BASENAMES_NOEXT) {
IO_RENAME_MAP[i] = INDEX_BASENAME_NOEXT;
}
exports.IO_RENAME_MAP = IO_RENAME_MAP;
function renameBasename(original) {
if (original in IO_RENAME_MAP) {
return IO_RENAME_MAP[original];
} else {
return original;
}
}
function renderError(context, message, source_location, severity=1) {
if (!context.ignore_errors) {
context.errors.push(new ErrorMessage(message, source_location, severity));
}
}
function renderErrorXUndefined(ast, context, target_id, options={}) {
let { source_location } = options
let message = `internal link ${ESCAPE_CHAR}${Macro.X_MACRO_NAME} to unknown ID: "${target_id}" at render time`;
if (source_location === undefined) {
if (ast.args.href) {
source_location = ast.args.href.source_location
} else {
source_location = ast.source_location
}
}
renderError(context, message, source_location, 2);
return errorMessageInOutput(message, context)
}
/** Render the ToC from a list representation rather than tree.
*
* This function was introduced to factor out the static CLI ToC and the dynamic one from Web,
* the static one had a tree representation, but the dynamic one has a list, so we convert
* both to a single list representation and render it here.
*/
function renderTocFromEntryList({
add_test_instrumentation,
context,
entry_list,
descendant_count_html,
hasSearch,
tocHasSelflinks,
tocIdPrefix,
}) {
let top_level = 0;
if (tocIdPrefix === undefined) {
tocIdPrefix = ''
}
if (hasSearch === undefined) {
hasSearch = true
}
if (tocHasSelflinks === undefined) {
// There is currently no code that sets this to true. Previously it was fixed at true,
// and we decided to drop it by default, but keep the code around just in case.
tocHasSelflinks = false
}
let ret = `<div class="toc-container" id="${tocIdPrefix}${Macro.TOC_ID}">` +
`<ul><li${htmlClassAttr([TOC_HAS_CHILD_CLASS, 'toplevel'])}>` +
`<div class="title-div">` +
`${TOC_ARROW_HTML}<span class="not-arrow">` +
`<a class="title toc" href="#${tocIdPrefix}${Macro.TOC_ID}"> Table of contents</a>`
if (hasSearch) {
ret += `<input class="search" placeholder="${UNICODE_SEARCH_CHAR} Search. Shortcut: / (slash)">`
}
if (descendant_count_html) {
ret += `<span class="hover-metadata">${descendant_count_html}</span>`
}
ret += `</span></div>`
for (let i = 0; i < entry_list.length; i++) {
const entry = entry_list[i]
let {
addLink,
content,
href,
level,
has_child,
id_prefix,
link_to_split,
parent_href,
parent_content,
target_id,
} = entry
if (id_prefix === undefined) {
id_prefix = ''
}
if (level > top_level) {
ret += `<ul>`;
} else if (level < top_level) {
ret += `</li></ul>`.repeat(top_level - level);
} else {
ret += `</li>`;
}
ret += '<li';
if (has_child) {
ret += htmlClassAttr([TOC_HAS_CHILD_CLASS]);
}
ret += '>'
const my_toc_id = tocIdWithScopeRemoval(target_id, context);
let id_to_toc = htmlAttr(Macro.ID_ARGUMENT_NAME, htmlEscapeAttr(my_toc_id));
let linear_count_str
if (add_test_instrumentation) {
linear_count_str = htmlAttr('data-test', i)
} else {
linear_count_str = ''
}
ret += `<div${id_to_toc}>${TOC_ARROW_HTML}` +
`<span class="not-arrow">` +
`<a${href}${linear_count_str}>${content}</a>` +
`<span class="hover-metadata">`
let toc_href = htmlAttr('href', '#' + htmlEscapeHrefAttr(my_toc_id));
if (tocHasSelflinks) {
// c for current
ret += `<a${toc_href}${htmlAttr('class', 'c')}></a>`
}
if (link_to_split) {
ret += `${link_to_split}`;
}
if (parent_href) {
// p for parent
ret += `<a${parent_href}${htmlAttr('class', 'u')}> ${parent_content}</a>`;
}
if (addLink) {
ret += addLink
}
if (entry.descendant_count_html) {
ret += `${entry.descendant_count_html}`
}
ret += `</span></span></div>`
top_level = level;
}
ret += `</li></ul>`.repeat(top_level);
// Close the table of contents list.
ret += `</li></ul>`;
ret += `</div>`
return ret
}
exports.renderTocFromEntryList = renderTocFromEntryList
function renderToc(context) {
if (context.toc_was_rendered || !checkHasToc(context)) {
// Empty ToC. Don't render. Initial common case: leaf split header nodes.
return '';
}
context.toc_was_rendered = true
let entry_list = []
// Not rendering ID here because that function does scope culling. But TOC ID is a fixed value without scope for now.
// so that was removing the TOC id in subdirectories.
let todo_visit = [];
let root_node = context.header_tree;
if (root_node.children.length === 1) {
root_node = root_node.children[0];
}
let descendant_count_html = getDescendantCountHtml(context, root_node, { long_style: false, show_descendant_count: true });
for (let i = root_node.children.length - 1; i >= 0; i--) {
todo_visit.push([root_node.children[i], 1]);
}
while (todo_visit.length > 0) {
const entry = {}
const [tree_node, level] = todo_visit.pop();
entry.level = level
const has_child = tree_node.children.length > 0
entry.has_child = has_child
let target_ast = context.db_provider.get(tree_node.ast.id, context);
if (
// Can happen in test error cases:
// - internal link from header title without ID to previous header is not allowed
// - include to file that does exists without embed includes before extracting IDs fails gracefully
target_ast !== undefined
) {
// I had this at one point, but it was too confusing that \x links linked to split, and ToC to nonsplit.
// If we want to keep split self contained, then we have to do it everywhere I think, not just ToC.
//// ToC entries always link to the same split/nosplit type, except for included sources.
//// This might be handled more generally through: https://github.com/ourbigbook/ourbigbook/issues/146
//// but for now we are just taking care of this specific and important ToC subcase.
//let cur_context;
//if (ast.source_location.path === target_ast.source_location.path) {
// cur_context = cloneAndSet(context, 'to_split_headers', context.in_split_headers);
//} else {
// cur_context = context;
//}
entry.content = xText(target_ast, context, {
addNumberDiv: true,
addNumberElem: 'i',
addNumberClass: 'n',
show_caption_prefix: false,
style_full: true,
});
entry.href = xHrefAttr(target_ast, context);
entry.target_id = target_ast.id
if (context.options.split_headers) {
entry.link_to_split = linkToSplitOpposite(target_ast, context)
}
let parent_ast = target_ast.get_header_parent_asts(context)[0];
if (
// Possible on broken h1 level.
parent_ast !== undefined
) {
let parent_href_target;
if (
parent_ast.header_tree_node !== undefined &&
parent_ast.header_tree_node.get_level() === context.header_tree_top_level
) {
parent_href_target = context.options.tocIdPrefix + Macro.TOC_ID;
} else {
parent_href_target = tocIdWithScopeRemoval(parent_ast.id, context);
}
entry.parent_href = htmlAttr('href', '#' + parent_href_target);
entry.parent_content = xText(parent_ast, context, { show_caption_prefix: false })
}
// The inner <div></div> inside arrow is so that:
// - outter div: takes up space to make clicking easy
// - inner div: minimal size to make the CSS arrow work, but too small for confortable clicking
entry.descendant_count_html = getDescendantCountHtml(context, tree_node, { long_style: false, show_descendant_count: true });
}
if (has_child) {
for (let i = tree_node.children.length - 1; i >= 0; i--) {
todo_visit.push([tree_node.children[i], level + 1]);
}
}
entry_list.push(entry)
}
return renderTocFromEntryList({
context,
entry_list,
descendant_count_html,
add_test_instrumentation: context.options.add_test_instrumentation,
tocIdPrefix: context.options.tocIdPrefix,
})
}
function perfPrint(context, name) {
// Includes and CirodowExample also call convert to parse.
// For now we are ignoring those recursions. A more correct approach
// would be to track the time intervals of those subconverts, and add
// them up to the corresponding toplevel convert.
if (!context.options.from_include) {
const now = globals.performance.now();
const delta = now - context.perf_prev
context.extra_returns.debug_perf[name] = now
context.perf_prev = now
if (context.options.log.perf || context.options.log.mem) {
console.error(`perf ${name} t=${now} dt=${delta}`);
if (context.options.log.mem) {
global.gc()
console.error(process.memoryUsage());
}
}
}
}
exports.perfPrint = perfPrint
function pluralizeWrap(s, n) {
let ret = pluralize(s, n)
if (n === undefined || n > 1 && s !== ret) {
// https://github.com/plurals/pluralize/issues/127
const last = ret[ret.length - 1]
if (last === 'S') {
ret = ret.substring(0, ret.length - 1) + last.toLowerCase()
}
}
return ret
}
exports.pluralizeWrap = pluralizeWrap
function propagateNumbered(ast, context) {
// numbered propagation to children.
// Note that the property only affects descendants, but not the node itself.
const parent_tree_node = ast.header_tree_node.parent_ast
if (
parent_tree_node === undefined ||
parent_tree_node.ast === undefined
) {
// Try getting parents from \Include.
// https://github.com/ourbigbook/ourbigbook/issues/188
//const parent_asts = ast.get_header_parent_asts(context)
//if (parent_asts.length > 0) {
// ast.numbered = parent_asts.some(ast => ast.numbered)
//} else {
ast.numbered = context.options.h.numbered
} else {
const parent_ast = parent_tree_node.ast
if (parent_ast.validation_output.numbered.given) {
ast.numbered = parent_ast.validation_output.numbered.boolean
} else {
ast.numbered = parent_ast.numbered
}
}
}
exports.propagateNumbered = propagateNumbered
// Fuck JavaScript? Can't find a built-in way to get the symbol string without the "Symbol(" part.
// https://stackoverflow.com/questions/30301728/get-the-description-of-a-es6-symbol
function symbolToString(symbol) {
return symbol.toString().slice(7, -1);
}
function titleToIdContext(title, options={}, context) {
return titleToId(
title,
{
keepScopeSep: options.keep_scope_sep,
magic: options.magic,
normalizeLatin:
(
options.normalize !== undefined &&
options.normalize.latin !== undefined &&
options.normalize.latin
) ||
(
context !== undefined &&
context.options.ourbigbook_json.id.normalize.latin
)
,
normalizePunctuation:
(
(
options.normalize !== undefined &&
options.normalize.punctuation !== undefined &&
options.normalize.punctuation
) ||
(
context !== undefined &&
context.options.ourbigbook_json.id.normalize.punctuation
)
)
,
removeLeadingAt: (
context !== undefined &&
context.options.x_remove_leading_at
),
}
)
}
const titleToId = runtime_common.titleToId
exports.titleToId = titleToId
/** Heuristic only. */
function idToTitle(id) {
return capitalizeFirstLetter(id).replaceAll(ID_SEPARATOR, ' ')
}
exports.idToTitle = idToTitle
/** mathematics/calculus/limit -> mathematics/calculus */
function idToScope(id) {
return id.split(Macro.HEADER_SCOPE_SEPARATOR).slice(0, -1).join(Macro.HEADER_SCOPE_SEPARATOR)
}
exports.idToScope = idToScope
/** Factored out calculations of the ID that is given to each TOC entry.
*
* For after everything broke down due to toplevel scope.
*/
function tocId(id) {
return Macro.TOC_PREFIX + id;
}
exports.tocId = tocId
function tocIdWithScopeRemoval(id, context) {
if (context && id !== undefined) {
id = removeToplevelScope(id, context.toplevel_ast, context)
}
return tocId(id)
}
function unconvertible(ast, context) {
const msg = `macro "${ast.macro_name}" must never render`
if (context.in_parse) {
renderError(context, msg, ast.source_location);
return errorMessageInOutput(msg, context)
} else {
throw new Error(msg);
}
}
function urlBasename(str) {
return basename(str, URL_SEP);
}
// Do some error checking and setup some stuff like boolean.
// We should likely do this in the AstNode constructor. The reason we didn't
// was likely to not need context at that point, and be nicer to serialization.
function validateAst(ast, context) {
if (ast.validated) {
throw new Error(`ast has already been validated:\n${ast.toString()}`)
} else {
ast.validated = true
}
const macro_name = ast.macro_name;
const macro = context.macros[macro_name];
const name_to_arg = macro.name_to_arg;
// First pass sets defaults or missing arguments.
for (const argname in name_to_arg) {
ast.validation_output[argname] = {};
const macro_arg = name_to_arg[argname];
if (argname in ast.args) {
ast.validation_output[argname].given = true;
const arg = ast.args[argname]
if (macro_arg.empty && arg.asts.length) {
ast.validation_error = [
`empty argument "${argname}" of macro "${macro_name}" was not empty: https://docs.ourbigbook.com/empty-macro-argument`,
arg.asts[0].source_location,
]
}
if (
macro_arg.disabled &&
!(
context.options.ourbigbook_json.enableArg &&
context.options.ourbigbook_json.enableArg[macro_name] &&
context.options.ourbigbook_json.enableArg[macro_name][argname]
)
) {
ast.validation_error = [
`disabled argument "${argname}" of macro "${macro_name}": ${OURBIGBOOK_DEFAULT_DOCS_URL}/disabled-macro-argument`,
arg.source_location,
];
}
} else {
ast.validation_output[argname].given = false;
if (macro_arg.mandatory) {
ast.validation_error = [
`missing mandatory argument ${argname} of ${ast.macro_name}`,
ast.source_location,
];
}
if (macro_arg.default !== undefined) {
ast.args[argname] = new AstArgument([
new PlaintextAstNode(macro_arg.default, ast.source_location)]);
} else if (macro_arg.boolean) {
ast.args[argname] = new AstArgument([new PlaintextAstNode(Macro.BOOLEAN_ARGUMENT_FALSE, ast.source_location)]);
}
}
}
// Second pass processes the values including defaults.
for (const argname in name_to_arg) {
const macro_arg = name_to_arg[argname];
if (argname in ast.args) {
const arg = ast.args[argname];
if (macro_arg.boolean) {
let arg_string;
if (arg.length() > 0) {
arg_string = renderArgNoescape(arg, cloneAndSet(context, 'id_conversion', true));
} else {
arg_string = Macro.BOOLEAN_ARGUMENT_TRUE;
}
if (arg_string === Macro.BOOLEAN_ARGUMENT_FALSE) {
ast.validation_output[argname].boolean = false;
} else if (arg_string === Macro.BOOLEAN_ARGUMENT_TRUE) {
ast.validation_output[argname].boolean = true;
} else {
ast.validation_output[argname].boolean = false;
ast.validation_error = [
`boolean argument "${argname}" of "${ast.macro_name}" has invalid value: "${arg_string}", only "${Macro.BOOLEAN_ARGUMENT_FALSE}" and "${Macro.BOOLEAN_ARGUMENT_TRUE}" are allowed`,
arg.source_location
];
break;
}
}
if (macro_arg.positive_nonzero_integer) {
const arg_string = renderArgNoescape(arg, cloneAndSet(context, 'id_conversion', true));
const intValue = parseNonNegativeIntegerStrict(arg_string);
ast.validation_output[argname].positive_nonzero_integer = intValue;
if (!Number.isInteger(intValue) || !(intValue > 0)) {
ast.validation_error = [
`argument "${argname}" of macro "${ast.macro_name}" must be a positive non-zero integer, got: "${arg_string}"`,
arg.source_location
];
break;
}
}
}
}
}
exports.validateAst = validateAst
function xChildDbEffectiveId(target_id, context, ast) {
const target_ast = context.db_provider.get(target_id, context, ast.scope);
if (
target_ast === undefined
) {
// Return as is. This can happen during ID extraction when the
// target ID needs to be resolved across directory based scopes.
// In those cases, the target ID could be on another file, so we would need to
// read the database to decide. But reading from database during ID extraction
// is forbidden, so we do that on a separate pass.
// Related: https://github.com/ourbigbook/ourbigbook/issues/229
if (isAbsoluteXref(target_id, context)) {
target_id = resolveAbsoluteXref(target_id, context)
}
return target_id
} else {
return target_ast.id
}
}
function xGetTargetAstBase({
context,
do_magic_title_to_id,
do_singularize,
scope,
target_id,
}) {
if (
context.options.x_leading_at_to_web &&
target_id[0] === AT_MENTION_CHAR
) {
return [htmlAttr('href', context.webUrl + target_id.substring(1)), target_id];
}
let target_id_eff
if (do_magic_title_to_id) {
target_id_eff = magicTitleToId(target_id, context)
} else {
target_id_eff = target_id
}
let target_ast = context.db_provider.get(target_id_eff, context, scope);
if (do_singularize && !target_ast) {
target_id_eff = magicTitleToId(pluralizeWrap(target_id, 1), context)
target_ast = context.db_provider.get(target_id_eff, context, scope);
}
return { target_id: target_id_eff, target_ast }
}
function convertFileIdArg(ast, href_arg, context) {
let target_id = convertIdArg(href_arg, context);
if (
ast.validation_output.file.given
) {
target_id = Macro.FILE_ID_PREFIX + target_id
}
return target_id
}
function xGetTargetAst(ast, context) {
const href_arg = ast.args.href
const target_id = convertFileIdArg(ast, href_arg, context);
const ret = xGetTargetAstBase({
context,
do_magic_title_to_id: ast.validation_output.magic.boolean,
do_singularize: ast.validation_output.magic.boolean,
scope: ast.scope,
target_id,
})
ret.href_arg = href_arg
ret.target_id_raw = target_id
return ret
}
/**
* @param {AstNode} ast \x ast node
* @return {[String, String]} [href, content] pair for the x node.
*/
function xGetHrefContent(ast, context, opts={}) {
const { showDisambiguate } = opts
const { href_arg, target_id, target_id_raw, target_ast } = xGetTargetAst(ast, context)
const content_arg = ast.args.content;
if (ast.validation_output.topic.boolean) {
let topicTitle = target_id_raw
let topicTitleSingular = topicTitle
if (!ast.validation_output.p.boolean) {
topicTitleSingular = pluralizeWrap(topicTitleSingular, 1)
}
const topicId = titleToIdContext(topicTitleSingular, undefined, context)
if (content_arg !== undefined) {
topicTitle = renderArg(content_arg, context)
}
return [htmlAttr('href', `${context.options.webMode ? URL_SEP : context.webUrl}${WEB_TOPIC_PATH}${URL_SEP}${topicId}`), topicTitle, undefined];
}
// href
let href
if (target_ast) {
href = xHrefAttr(target_ast, context)
}
// content
let content;
if (content_arg === undefined) {
if (target_id === context.options.ref_prefix) {
content = HTML_HOME_MARKER
} else {
if (context.renderXAsHref) {
content = renderArg(href_arg, context)
} else {
if (!target_ast) {
return [href, renderErrorXUndefined(ast, context, target_id)];
}
let x_text_options = {
caption_prefix_span: false,
capitalize: ast.validation_output.c.boolean,
from_x: true,
quote: true,
pluralize: ast.validation_output.p.given ? ast.validation_output.p.boolean : undefined,
};
if (ast.validation_output.magic.boolean) {
const first_ast = href_arg.get(0);
if (first_ast.node_type === AstType.PLAINTEXT) {
const sep_idx = first_ast.text.lastIndexOf(Macro.HEADER_SCOPE_SEPARATOR)
const idx = sep_idx === -1 ? 0 : sep_idx + 1
const c = first_ast.text[idx]
if (
// Possible on home article on web, which has empty ID.
c !== undefined &&
c !== c.toLowerCase()
) {
x_text_options.capitalize = true
}
}
const last_ast = href_arg.get(href_arg.length() - 1);
if (last_ast.node_type === AstType.PLAINTEXT) {
const text = first_ast.text
if (
text !== pluralizeWrap(text, 1) &&
// Due to buggy pluralize behaviour, it can be different from both.
// So let's check and abort otherwise just using what is actually in the ref
// https://github.com/plurals/pluralize/issues/172
// for those weirder cases.
text === pluralizeWrap(text, 2)
) {
x_text_options.pluralize = true
}
if (ast.validation_output.magic.boolean) {
const words = text.split(PLURALIZE_WORD_SPLIT_REGEX)
x_text_options.forceLastWord = words[words.length - 1]
}
}
}
if (ast.validation_output.full.given) {
x_text_options.style_full = ast.validation_output.full.boolean
x_text_options.style_full_from_x = true
}
const xTextBaseRet = xTextBase(target_ast, context, x_text_options);
if (showDisambiguate) {
content = xTextBaseRet.innerWithDisambiguate
} else {
content = xTextBaseRet.full
}
if (content === ``) {
let message = `empty internal link body: "${target_id}"`;
renderError(context, message, ast.source_location);
return errorMessageInOutput(message, context);
}
}
}
} else {
// Explicit content given, just use it then.
content = renderArg(content_arg, context);
}
return [href, content, target_ast];
}
/** Calculate the href value to a given target AstNode.
*
* This takes into account e.g. if the target node is in a different source file:
* https://docs.ourbigbook.com#internal-cross-file-references
*
* @param {AstNode} target_ast
* @return {String} the value of href (no quotes) that an \x internal link to the given target_ast
*/
function xHref(target_ast, context) {
const [href_path, fragment] = xHrefParts(target_ast, context);
let ret = href_path;
if (fragment !== '')
ret += '#' + fragment;
return ret;
}
function indexPathFromDirname(dirname_ret, basename_ret, sep) {
const [dir_dir, dir_base] = pathSplit(dirname_ret, sep);
if (basename_ret === INDEX_BASENAME_NOEXT && dirname_ret) {
dirname_ret = dir_dir
basename_ret = dir_base
}
return [dirname_ret, basename_ret]
}
function isPunctuation(c) {
return c === '.' ||
c === '!' ||
c === '?' ||
c === ')'
}
// Get the path to the split header version
//
// to_split_headers is set explicitly when making
// links across split/non-split versions of the output.
//
// Otherwise, link to the same type of output as the current one
// as given in in_split_headers.
//
// This way, to_split_hedears in particular forces the link to be
// to the non-split mode, even if we are in split mode.
//
// Some desired sample outcomes:
//
// id='ourbigbook' -> ['', 'index-split']
// id='quick-start' -> ['', 'quick-start']
// id='not-index' -> ['', 'not-index-split']
// id='h2-in-not-the-index' -> ['', 'h2-in-not-the-index']
// id='subdir' -> ['subdir', 'index-split']
// id='subdir/subdir-h2' -> ['subdir', 'subdir-h2']
// id='subdir/notindex' -> ['subdir', 'notindex']
// id='subdir/notindex-h2' -> ['subdir', 'notindex-h2']
function isToSplitHeaders(ast, context) {
return isToSplitHeadersBase(
ast.split_default,
context.to_split_headers,
context.options.ourbigbook_json.toSplitHeaders,
)
}
function isToSplitHeadersBase(
ast_split_default,
context_to_split_headers,
to_split_headers_override,
) {
return (to_split_headers_override !== undefined && to_split_headers_override) ||
(context_to_split_headers === undefined && ast_split_default) ||
(context_to_split_headers !== undefined && context_to_split_headers);
}
/** This is the centerpiece of x href calculation!
*
* This code is crap. There are too many cases for my brain to handle.
* So I just write tests, and hack the code until the tests pass, but
* I'm not capable of factoring it nicely.
*
* @param {AstNode} target_ast
*/
function xHrefParts(target_ast, context) {
// Synonym handling.
let target_ast_effective_id, first_toplevel_child_effective
if (target_ast.synonym === undefined) {
target_ast_effective_id = target_ast.id
first_toplevel_child_effective = target_ast.first_toplevel_child
} else {
const synonym_target_ast = context.db_provider.get(target_ast.synonym, context);
target_ast_effective_id = synonym_target_ast.id
first_toplevel_child_effective = synonym_target_ast.first_toplevel_child
}
let to_split_headers = isToSplitHeaders(target_ast, context);
let to_current_toplevel =
target_ast_effective_id === context.toplevel_id &&
// Also requires outputting to the same type of split/nonsplit
// as the current one.
context.in_split_headers === to_split_headers
;
// href_path
let href_path,
toplevel_output_path_dirname = '',
toplevel_output_path_basename,
full_output_path,
target_output_path_dirname,
target_output_path_basename,
split_suffix
;
const target_input_path = target_ast.source_location.path;
if (
target_ast.source_location.path === undefined ||
context.toplevel_output_path === undefined ||
(
// Nosplit header link to a header that renders on the
// same page.
!context.in_split_headers &&
!(
context.to_split_headers !== undefined &&
context.to_split_headers
) &&
context.toplevel_id === target_ast.toplevel_id &&
// https://github.com/ourbigbook/ourbigbook/issues/365
!context.options.webMode
) ||
(
// Split header link to image in current header.
context.in_split_headers &&
target_ast.macro_name !== Macro.HEADER_MACRO_NAME &&
target_ast.get_header_parent_ids(context).has(context.toplevel_id)
) ||
(
to_current_toplevel &&
// https://github.com/ourbigbook/ourbigbook/issues/365
!context.options.webMode
)
) {
// The output path is the same as the current path. Stop.
// Everything else is basically handled by outputPathBase.
// That function doesn't have the concept of "where am I looking from".
// so we do that here. Maybe it would be better to change that, if if
// is hair tearing.
href_path = '';
} else {
;[toplevel_output_path_dirname, toplevel_output_path_basename] =
pathSplit(context.toplevel_output_path, context.options.path_sep);
;({
path: full_output_path,
dirname: target_output_path_dirname,
basename: target_output_path_basename,
split_suffix,
} = target_ast.output_path(
context,
{
effective_id: target_ast_effective_id,
toSplitHeadersOverride: context.options.ourbigbook_json.toSplitHeaders,
}
));
if (context.options.x_remove_leading_at) {
if (target_output_path_dirname) {
if (target_output_path_dirname[0] === AT_MENTION_CHAR) {
target_output_path_dirname = target_output_path_dirname.slice(1)
}
} else {
if (
target_output_path_basename &&
target_output_path_basename[0] === AT_MENTION_CHAR
) {
target_output_path_basename = target_output_path_basename.slice(1)
}
}
}
// The target path is the same as the current path being output.
if (
full_output_path === context.toplevel_output_path &&
// https://github.com/ourbigbook/ourbigbook/issues/365
!context.options.webMode
) {
href_path = ''
} else {
let href_path_dirname_rel
if (context.options.x_absolute) {
href_path_dirname_rel = target_output_path_dirname
} else {
href_path_dirname_rel = path.relative(
toplevel_output_path_dirname, target_output_path_dirname);
}
if (
// Same output path.
href_path_dirname_rel === '' &&
target_output_path_basename === toplevel_output_path_basename
) {
target_output_path_basename = '';
} else {
if (context.options.htmlXExtension) {
target_output_path_basename += '.' + HTML_EXT;
} else if (target_output_path_basename === INDEX_BASENAME_NOEXT) {
if (href_path_dirname_rel === '') {
target_output_path_basename = '.';
} else {
target_output_path_basename = '';
}
}
}
href_path = pathJoin(href_path_dirname_rel,
target_output_path_basename, context.options.path_sep);
}
}
if (!context.options.include_path_set.has(target_input_path)) {
href_path = context.options.x_external_prefix + href_path
}
if (
context.options.ourbigbook_json.xPrefix !== undefined &&
href_path !== '' &&
split_suffix === undefined
) {
href_path = context.options.ourbigbook_json.xPrefix + pathJoin(
toplevel_output_path_dirname, href_path, context.options.path_sep)
}
if (href_path && context.options.x_absolute) {
href_path = '/' + href_path
}
// Fragment
if (
!context.in_split_headers &&
context.options.include_path_set.has(target_input_path) &&
!(context.to_split_headers !== undefined && context.to_split_headers)
) {
// We are not in split headers, and the output is in the current file.
// Therefore, don't use the split header target no matter what its
// splitDefault is, use the non-split one.
to_split_headers = false;
}
let fragment;
if (
(
target_ast.macro_name === Macro.HEADER_MACRO_NAME &&
(
// Linking to a toplevel ID.
first_toplevel_child_effective ||
// Linking towards a split header not included in the current output.
to_split_headers ||
to_current_toplevel ||
target_ast.id === target_ast.toplevel_id
)
)
) {
// An empty href means the beginning of the page.
fragment = '';
} else {
// This is the AST that will show up on the top of the rendered page.
// that contains target_ast. We need to know it for toplevel scope culling.
let toplevel_ast;
if (to_split_headers) {
// We know not a header target, as that would have been caught previously.
toplevel_ast = target_ast.get_header_parent_asts(context)[0];
} else {
if (target_ast.toplevel_id === undefined) {
toplevel_ast = context.nosplit_toplevel_ast
} else {
toplevel_ast = context.db_provider.get(target_ast.toplevel_id, context);
}
//const get_file_ret = context.options.db_provider.get_file(target_input_path);
//if (get_file_ret) {
//} else {
// // The only way this can happen is if we are in the current file, and it hasn't
// // been added to the file db yet.
//}
}
fragment = removeToplevelScope(target_ast_effective_id, toplevel_ast, context);
}
// return
return [htmlEscapeHrefAttr(href_path), htmlEscapeAttr(fragment)];
}
/* href="" that links to a given node. */
function xHrefAttr(target_ast, context) {
return htmlAttr('href', xHref(target_ast, context));
}
/**
* Calculate the text (visible content) of a internal link, or the text
* that the caption text that internal links can refer to, e.g.
* "Table 123. My favorite table". Both are done in a single function
* so that style_full references will show very siimlar to the caption
* they refer to.
*
* @param {Object} options
* @param {Object} href_prefix rendered string containing the href="..."
* part of a link to self to be applied e.g. to <>Figure 1<>, of undefined
* if this link should not be given.
* @return {string} full: '<a href="#barack-obama/equation-my-favorite-equation"><span class="caption-prefix">Equation 1</span></a>. My favorite equation'
* {string} inner: 'My favorite equation'
* {string} innerWithDisambiguate: 'Python (programming language)'. Note no .meta span on the disambiguate.
*/
function xTextBase(ast, context, options={}) {
context = cloneAndSet(context, 'in_a', true)
if (!('addNumberDiv' in options)) {
options.addNumberDiv = false;
}
if (!('addTitleDiv' in options)) {
options.addTitleDiv = false;
}
if (!('caption_prefix_span' in options)) {
options.caption_prefix_span = true;
}
if (!('quote' in options)) {
options.quote = false;
}
if (!('fixed_capitalization' in options)) {
options.fixed_capitalization = true;
}
if (!('href_prefix' in options)) {
options.href_prefix = undefined;
}
if (!('force_separator' in options)) {
options.force_separator = false;
}
if (!('from_x' in options)) {
options.from_x = false;
}
if (!('pluralize' in options)) {
// true: make plural
// false: make singular
// undefined: don't touch it
options.pluralize = undefined;
}
if (!('show_caption_prefix' in options)) {
options.show_caption_prefix = true;
}
let forceLastWord
if ('forceLastWord' in options) {
forceLastWord = options.forceLastWord
} else {
forceLastWord = undefined
}
const macro = context.macros[ast.macro_name];
let inner
let innerWithDisambiguate
let innerNoDiv
let { style_full, style_full_from_x } = options
if (style_full === undefined) {
style_full = macro.options.default_x_style_full
}
if (style_full_from_x === undefined) {
// Comes from a {full} argument of actual X argument and not e.g. a main h1 header render.
// These are treated differently: the full header render
// - shows title 2
// - has <span class="meta"> wrapper for the different color
style_full_from_x = false
}
let ret = ``;
let title_arg = macro.options.get_title_arg(ast, context);
if (style_full) {
if (options.href_prefix !== undefined) {
ret += `<a${options.href_prefix}>`
}
if (options.show_caption_prefix) {
if (options.caption_prefix_span) {
ret += `<span class="caption-prefix">`;
}
ret += `${macro.options.caption_prefix} `;
}
if (
ast.numbered &&
(
// When in split headers, numbers are only added to headers that
// are descendants of the toplevel header, thus matching the current ToC.
// The numbers don't make much sense for other headers.
ast.macro_name !== Macro.HEADER_MACRO_NAME ||
(
// Possible in case of broken header parent=.
context.toplevel_ast !== undefined &&
ast.is_header_local_descendant_of(context.toplevel_ast, context)
)
)
) {
let number = macro.options.get_number(ast, context);
if (number !== undefined) {
if (
(
(title_arg !== undefined && style_full) ||
options.force_separator
)
) {
number += '. '
}
if (options.addNumberDiv) {
const addNumberElem = options.addNumberElem ? options.addNumberElem : 'span'
const addNumberClass = options.addNumberClass ? options.addNumberClass : 'number'
number = `<${addNumberElem} class="${addNumberClass}">${number}</${addNumberElem}>`
}
ret += number;
}
}
if (options.show_caption_prefix && options.caption_prefix_span) {
ret += `</span>`;
}
if (options.href_prefix !== undefined) {
ret += `</a>`
}
}
if (
title_arg !== undefined
) {
if (style_full && options.quote) {
ret += htmlEscapeContext(context, `"`);
}
if (
ast.id === context.options.ref_prefix &&
!context.renderFirstHeaderNotAsHome
) {
innerNoDiv = HTML_HOME_MARKER
} else {
context.renderFirstHeaderNotAsHome = false
// Hack title_arg with {c} and {p} corrections:
// https://docs.ourbigbook.com#internal-link-title-inflection
if (options.from_x) {
// {c}
let first_ast = title_arg.get(0);
if (
ast.macro_name === Macro.HEADER_MACRO_NAME &&
!ast.validation_output.c.boolean &&
!style_full &&
first_ast.node_type === AstType.PLAINTEXT
) {
// https://stackoverflow.com/questions/41474986/how-to-clone-a-javascript-es6-class-instance
title_arg = lodash.clone(title_arg)
title_arg.asts = lodash.clone(title_arg.asts)
// This is not amazing, as it can change some properties of the node,
// as we are not copying anything besides text and source_location.
// This almost had an impact on {toplevel} rendering, but it dien't matter
// so we didn't try to sanitize it further for now.
title_arg.set(0, new PlaintextAstNode(first_ast.text, first_ast.source_location))
let txt = title_arg.get(0).text;
let first_c = txt[0];
if (options.capitalize) {
first_c = first_c.toUpperCase();
} else {
first_c = first_c.toLowerCase();
}
title_arg.get(0).text = first_c + txt.substring(1);
}
// {p}
let last_ast = title_arg.get(title_arg.length() - 1);
if (
options.pluralize !== undefined &&
!style_full &&
last_ast.node_type === AstType.PLAINTEXT
) {
title_arg = lodash.clone(title_arg)
title_arg.asts = lodash.clone(title_arg.asts)
let text = forceLastWordReplace(last_ast.text, forceLastWord)
title_arg.set(
title_arg.length() - 1,
new PlaintextAstNode(
pluralizeWrap(text, options.pluralize ? 2 : 1), last_ast.source_location
)
)
}
}
if (ast.file) {
innerNoDiv = ast.file
} else {
if (ast.macro_name === Macro.HEADER_MACRO_NAME) {
context = cloneAndSet(context, 'renderXAsHref', true)
}
innerNoDiv = renderArg(title_arg, context);
}
}
if (options.addTitleDiv) {
inner = `<div class="title">${innerNoDiv}</div>`
} else {
inner = innerNoDiv
}
ret += inner
const disambiguate_arg = ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME];
const show_disambiguate = (disambiguate_arg !== undefined) && macro.options.show_disambiguate;
let disambiguateRender
if (show_disambiguate) {
disambiguateRender = renderArg(disambiguate_arg, context)
}
if (style_full) {
const title2_arg = ast.args[Macro.TITLE2_ARGUMENT_NAME];
const title2_renders = [];
if (show_disambiguate) {
title2_renders.push(disambiguateRender)
}
if (!style_full_from_x) {
if (title2_arg !== undefined) {
for (const arg of title2_arg.asts.map(ast => ast.args[Macro.CONTENT_ARGUMENT_NAME])) {
if (arg.asts.length) {
title2_renders.push(renderArg(arg, context));
}
}
}
for (const title2ast of ast.title2s) {
title2_renders.push(renderArg(title2ast.args.title, context));
}
}
if (title2_renders.length) {
let title2Render = `(${title2_renders.join(', ')})`
if (!style_full_from_x) {
title2Render = `<span class="meta">${title2Render}</span>`
}
ret += ` ${title2Render}`
}
if (options.quote) {
ret += htmlEscapeContext(context, `"`);
}
}
innerWithDisambiguate = inner
if (show_disambiguate) {
innerWithDisambiguate += ` (${disambiguateRender})`
}
}
return { full: ret, inner, innerWithDisambiguate, innerNoDiv }
}
function xText(ast, context, options={}) {
return xTextBase(ast, context, options).full
}
// consts
const AUTOMATIC_TOPIC_LINK_PUNCTUATION_REGEX_STR = '[.,:;!?"]+'
const AUTOMATIC_TOPIC_LINK_PUNCTUATION_REGEX = new RegExp(AUTOMATIC_TOPIC_LINK_PUNCTUATION_REGEX_STR)
const AUTOMATIC_TOPIC_LINK_PUNCTUATION_REGEX_CAPTURE = new RegExp(`(${AUTOMATIC_TOPIC_LINK_PUNCTUATION_REGEX_STR})`)
const ANCESTORS_ID_UNRESERVED = 'ancestors'
const ANCESTORS_ID = `${Macro.RESERVED_ID_PREFIX}${ANCESTORS_ID_UNRESERVED}`
exports.ANCESTORS_ID = ANCESTORS_ID
const ANCESTORS_MAX = 6
exports.ANCESTORS_MAX = ANCESTORS_MAX
const AT_MENTION_CHAR = runtime_common.AT_MENTION_CHAR
exports.AT_MENTION_CHAR = AT_MENTION_CHAR;
const H_ANCESTORS_CLASS = 'ancestors'
exports.H_ANCESTORS_CLASS = H_ANCESTORS_CLASS
const H_WEB_CLASS = 'web'
exports.H_WEB_CLASS = H_WEB_CLASS
// The container of an ourbigbook render should have this class.
const OURBIGBOOK_CSS_CLASS = 'ourbigbook'
exports.OURBIGBOOK_CSS_CLASS = OURBIGBOOK_CSS_CLASS
const FILE_ROOT_PLACEHOLDER = '(root)'
exports.FILE_ROOT_PLACEHOLDER = FILE_ROOT_PLACEHOLDER
const HTML_REF_MARKER = '<sup class="ref">[ref]</sup>'
const SHORTHAND_TOPIC_CHAR = '#';
exports.SHORTHAND_TOPIC_CHAR = SHORTHAND_TOPIC_CHAR
const WEB_API_PATH = 'api';
exports.WEB_API_PATH = WEB_API_PATH;
const WEB_TOPIC_PATH = 'go/topic';
exports.WEB_TOPIC_PATH = WEB_TOPIC_PATH
const PARAGRAPH_SEP = '\n\n';
exports.PARAGRAPH_SEP = PARAGRAPH_SEP;
const PLURALIZE_WORD_SPLIT_REGEX = /[^a-zA-Z]/
const REFS_TABLE_PARENT = 'PARENT';
exports.REFS_TABLE_PARENT = REFS_TABLE_PARENT;
const REFS_TABLE_X = 'X';
exports.REFS_TABLE_X = REFS_TABLE_X;
const REFS_TABLE_X_CHILD = 'X_CHILD';
exports.REFS_TABLE_X_CHILD = REFS_TABLE_X_CHILD;
// https://github.com/ourbigbook/ourbigbook/issues/198
const REFS_TABLE_X_TITLE_TITLE = 'X_TITLE_TITLE';
exports.REFS_TABLE_X_TITLE_TITLE = REFS_TABLE_X_TITLE_TITLE;
// Header is synonym of another one.
const REFS_TABLE_SYNONYM = 'SYNONYM';
exports.REFS_TABLE_SYNONYM = REFS_TABLE_SYNONYM;
const END_NAMED_ARGUMENT_CHAR = '}';
const END_POSITIONAL_ARGUMENT_CHAR = ']';
const ESCAPE_CHAR = '\\'
exports.ESCAPE_CHAR = ESCAPE_CHAR
// Rationale: 1 line = 80 characters.
// We want to preview files that are up to about 25 lines.
// More than that is wasteful in visual vertical area and bandwidth.
// Ideally we could just link to the files like images, but iframe does
// not work well, e.g. files that would be downloaded like .yml are also
// downloaded from the iframe
const FILE_PREVIEW_MAX_SIZE = 2000;
exports.FILE_PREVIEW_MAX_SIZE = FILE_PREVIEW_MAX_SIZE
const HEADER_PARENT_ERROR_MESSAGE = 'header parent either is a previous ID of a level, a future ID, or an invalid ID: '
const HTML_ASCII_WHITESPACE = new Set([' ', '\r', '\n', '\f', '\t']);
const HTML_EXT = 'html';
exports.HTML_EXT = HTML_EXT;
const INCOMING_LINKS_MARKER = '<span title="Incoming links" class="fa-solid-900 icon">\u{f060}</span>'
exports.INCOMING_LINKS_MARKER = INCOMING_LINKS_MARKER
const SYNONYM_LINKS_MARKER = '<span title="Synonyms" class="fa-solid-900 icon">\u{f07e}</span>'
exports.SYNONYM_LINKS_MARKER = SYNONYM_LINKS_MARKER
const TXT_HOME_MARKER = 'Home'
exports.TXT_HOME_MARKER = TXT_HOME_MARKER
const HTML_HOME_MARKER = `<span title="${TXT_HOME_MARKER}" class="fa-solid-900 icon">\u{f015}</span> ${TXT_HOME_MARKER}`
exports.HTML_HOME_MARKER = HTML_HOME_MARKER
const INCOMING_LINKS_ID_UNRESERVED = 'incoming-links'
exports.INCOMING_LINKS_ID_UNRESERVED = INCOMING_LINKS_ID_UNRESERVED
const SYNONYM_LINKS_ID_UNRESERVED = 'synonyms'
exports.SYNONYM_LINKS_ID_UNRESERVED = SYNONYM_LINKS_ID_UNRESERVED
const ID_SEPARATOR = runtime_common.ID_SEPARATOR
exports.ID_SEPARATOR = ID_SEPARATOR
const SHORTHAND_LIST_START = '* ';
const SHORTHAND_TD_START = '| ';
const SHORTHAND_TH_START = '|| ';
const SHORTHAND_LIST_INDENT = ' ';
const SHORTHAND_HEADER_CHAR = '=';
exports.SHORTHAND_HEADER_CHAR = SHORTHAND_HEADER_CHAR
const SHORTHAND_QUOTE_CHAR = '>'
const SHORTHAND_QUOTE_START = `${SHORTHAND_QUOTE_CHAR} `
exports.SHORTHAND_QUOTE_START = SHORTHAND_QUOTE_START
const LOG_OPTIONS = new Set([
'ast-inside',
'ast-inside-simple',
'ast-pp-simple',
'mem',
'parse',
'perf',
'split-headers',
'tokens-inside',
'tokenize',
'tokens',
]);
exports.LOG_OPTIONS = LOG_OPTIONS;
const IMAGE_EXTENSIONS = new Set([
'bmp',
'gif',
'jpeg',
'jpg',
'png',
'svg',
'tiff',
'webp',
])
const MULTILINE_CAPTION_CLASS = 'multiline'
const OURBIGBOOK_JSON_BASENAME = 'ourbigbook.json';
exports.OURBIGBOOK_JSON_BASENAME = OURBIGBOOK_JSON_BASENAME
const OURBIGBOOK_DEFAULT_HOST = 'ourbigbook.com'
exports.OURBIGBOOK_DEFAULT_HOST = OURBIGBOOK_DEFAULT_HOST
const OURBIGBOOK_DEFAULT_DOCS_URL = `https://docs.${OURBIGBOOK_DEFAULT_HOST}`
const OPTION_DEFAULTS = {
embed_includes: false,
htmlXExtension: true,
h: {
numbered: false,
},
// If true, this means that this is a ourbigbook --publish run rather
// than a regular development compilation.
publish: false,
}
const OURBIGBOOK_JSON_DEFAULT = {
dontIgnore: [],
dontIgnoreConvert: [],
enableArg: {},
fromOurBigBookExample: false,
generateSitemap: false,
h: {
splitDefault: false,
splitDefaultNotToplevel: false,
},
id: {
normalize: {
latin: true,
punctuation: true,
},
},
ignore: [],
ignoreConvert: [],
lint: {
startsWithH1Header: false,
filesAreIncluded: true,
'h-tag': undefined,
'h-parent': undefined,
},
openLinksOnNewTabs: false,
redirects: [],
publishRootUrl: undefined,
'unsafeXss': false,
web: {
host: OURBIGBOOK_DEFAULT_HOST,
hostCapitalized: 'OurBigBook.com',
link: false,
linkFromStaticHeaderMetaToWeb: false,
username: undefined,
},
xPrefix: undefined,
}
exports.OURBIGBOOK_JSON_DEFAULT = OURBIGBOOK_JSON_DEFAULT
const OUTPUT_FORMAT_OURBIGBOOK = 'bigb';
exports.OUTPUT_FORMAT_OURBIGBOOK = OUTPUT_FORMAT_OURBIGBOOK
const RENDER_TYPE_WEB = 'web'
exports.RENDER_TYPE_WEB = RENDER_TYPE_WEB
const OUTPUT_FORMAT_HTML = 'html';
exports.OUTPUT_FORMAT_HTML = OUTPUT_FORMAT_HTML
const OUTPUT_FORMAT_ID = 'id';
exports.OUTPUT_FORMAT_ID = OUTPUT_FORMAT_ID
const TOPLEVEL_INDEX_ID = '';
exports.TOPLEVEL_INDEX_ID = TOPLEVEL_INDEX_ID
const VIDEO_EXTENSIONS = new Set([
'avi',
'mkv',
'mov',
'mp4',
'ogv',
'webm',
])
const TAGGED_ID_UNRESERVED = 'tagged'
exports.TAGGED_ID_UNRESERVED = TAGGED_ID_UNRESERVED
const TAGS_MARKER = '<span title="Tags" class="fa-solid-900 icon">\u{f02c}</span>'
exports.TAGS_MARKER = TAGS_MARKER
const TOC_ARROW_HTML = '<div class="arrow"><div></div></div>'
const TOC_LINK_ELEM_CLASS_NAME = 'toc'
exports.TOC_LINK_ELEM_CLASS_NAME = TOC_LINK_ELEM_CLASS_NAME
const TOC_HAS_CHILD_CLASS = 'has-child'
const UL_OL_OPTS = {
wrapAttrs: { 'class': 'list' },
wrap: true,
}
const SHORTHAND_X_START = '<';
const SHORTHAND_X_END = '>';
const SHORTHAND_CODE_CHAR = '`'
const SHORTHAND_MATH_CHAR = '$'
const MAGIC_CHAR_ARGS = {
[SHORTHAND_MATH_CHAR]: Macro.MATH_MACRO_NAME,
[SHORTHAND_CODE_CHAR]: Macro.CODE_MACRO_NAME,
[SHORTHAND_X_START]: Macro.X_MACRO_NAME,
}
const NAMED_ARGUMENT_EQUAL_CHAR = '=';
const START_NAMED_ARGUMENT_CHAR = '{';
exports.START_NAMED_ARGUMENT_CHAR = START_NAMED_ARGUMENT_CHAR;
const START_POSITIONAL_ARGUMENT_CHAR = '[';
const SHORTHAND_LINK_END_CHARS = new Set([
' ',
'\n',
START_POSITIONAL_ARGUMENT_CHAR,
START_NAMED_ARGUMENT_CHAR,
END_POSITIONAL_ARGUMENT_CHAR,
END_NAMED_ARGUMENT_CHAR,
]);
const SHORTHAND_STARTS_TO_MACRO_NAME = {
[SHORTHAND_LIST_START]: Macro.LIST_ITEM_MACRO_NAME,
[SHORTHAND_QUOTE_START]: Macro.QUOTE_MACRO_NAME,
[SHORTHAND_TD_START]: Macro.TD_MACRO_NAME,
[SHORTHAND_TH_START]: Macro.TH_MACRO_NAME,
};
const MUST_ESCAPE_CHARS_REGEX_CHAR_CLASS = [
`\\${ESCAPE_CHAR}`,
START_POSITIONAL_ARGUMENT_CHAR,
`\\${END_POSITIONAL_ARGUMENT_CHAR}`,
START_NAMED_ARGUMENT_CHAR,
END_NAMED_ARGUMENT_CHAR,
SHORTHAND_X_START,
SHORTHAND_CODE_CHAR,
SHORTHAND_MATH_CHAR,
SHORTHAND_TOPIC_CHAR,
// \br
'\n',
].join('')
// https://docs.ourbigbook.com#known-url-protocols
const KNOWN_URL_PROTOCOL_NAMES = ['http', 'https'];
const KNOWN_URL_PROTOCOLS = new Set()
for (const name of KNOWN_URL_PROTOCOL_NAMES) {
KNOWN_URL_PROTOCOLS.add(name + '://');
}
const MUST_ESCAPE_CHARS_REGEX_CHAR_CLASS_REGEX = new RegExp(`([${MUST_ESCAPE_CHARS_REGEX_CHAR_CLASS}]|${Array.from(KNOWN_URL_PROTOCOLS).join('|')})`, 'g')
const MUST_ESCAPE_CHARS_AT_START_REGEX_CHAR_CLASS = [
'\\* ',
'=',
'\\|\\|',
'\\|',
`\\${SHORTHAND_QUOTE_CHAR}`,
].join('|')
const MUST_ESCAPE_CHARS_AT_START_REGEX_CHAR_CLASS_REGEX = new RegExp(`(^|\n)(${MUST_ESCAPE_CHARS_AT_START_REGEX_CHAR_CLASS})`, 'g')
const SHORTHAND_STARTS_MACRO_NAMES = new Set(Object.values(SHORTHAND_STARTS_TO_MACRO_NAME))
const AstType = makeEnum([
// An in-output error message.
'ERROR',
// The most regular and non-magic nodes.
// Most nodes are of this type.
'MACRO',
// A node that contains only text, and no subnodes.
'PLAINTEXT',
// Paragraphs are basically MACRO, but with some special
// magic because of the double newline madness treatment.
'PARAGRAPH',
'NEWLINE',
]);
const TokenType = makeEnum([
'INPUT_END',
'NEWLINE',
'MACRO_NAME',
'NAMED_ARGUMENT_END',
'NAMED_ARGUMENT_NAME',
'NAMED_ARGUMENT_START',
'PARAGRAPH',
'PLAINTEXT',
'POSITIONAL_ARGUMENT_END',
'POSITIONAL_ARGUMENT_START',
]);
const DEFAULT_MEDIA_HEIGHT = 315;
// Arguments for \Image, \image and \Video
const IMAGE_VIDEO_INLINE_BLOCK_NAMED_ARGUMENTS = [
new MacroArgument({
name: 'border',
boolean: true,
}),
new MacroArgument({
name: 'external',
boolean: true,
}),
new MacroArgument({
name: 'height',
default: DEFAULT_MEDIA_HEIGHT.toString(),
positive_nonzero_integer: true,
}),
new MacroArgument({
name: 'provider',
}),
new MacroArgument({
name: 'width',
positive_nonzero_integer: true,
}),
]
// Arguments for \Image and \image
const IMAGE_INLINE_BLOCK_NAMED_ARGUMENTS = [
new MacroArgument({
name: 'link',
elide_link_only: true,
}),
]
// Arguments for \Image and \Video
const IMAGE_VIDEO_BLOCK_NAMED_ARGUMENTS = IMAGE_VIDEO_INLINE_BLOCK_NAMED_ARGUMENTS.concat([
new MacroArgument({
name: Macro.TITLE_ARGUMENT_NAME,
count_words: true,
}),
new MacroArgument({
name: Macro.DESCRIPTION_ARGUMENT_NAME,
count_words: true,
}),
new MacroArgument({
name: 'source',
elide_link_only: true,
}),
new MacroArgument({
name: 'titleFromSrc',
boolean: true,
}),
]);
const XSS_SAFE_ALT_DEFAULT = {
[OUTPUT_FORMAT_HTML]: (ast, context) => {
return '<div>HTML snippet:</div>' + htmlCode(renderArg(ast.args[Macro.CONTENT_ARGUMENT_NAME], context))
}
}
const UNICODE_SEARCH_CHAR = '\u{1F50D}'
exports.UNICODE_SEARCH_CHAR = UNICODE_SEARCH_CHAR
/**
* Calculate a bunch of default parameters of the media from smart defaults if not given explicitly
*
* @return {Object}
* MediaProviderType {MediaProviderType} , e.g. type, src, source.
*/
function macroImageVideoResolveParams(ast, context) {
let error_message;
let media_provider_type;
let src = renderArgNoescape(ast.args.src, context);
let is_url;
// Provider explicitly given by user on macro.
if (ast.validation_output.provider.given) {
const provider_name = renderArgNoescape(ast.args.provider, cloneAndSet(context, 'id_conversion', true));
if (MEDIA_PROVIDER_TYPES.has(provider_name)) {
media_provider_type = provider_name;
} else {
error_message = `unknown media provider: "${htmlEscapeAttr(provider_name)}"`;
renderError(context, error_message, ast.args.provider.source_location);
media_provider_type = 'unknown';
}
}
// Otherwise, detect the media provider.
let media_provider_type_detected;
if (src.match(media_provider_type_wikimedia_re)) {
media_provider_type_detected = 'wikimedia';
} else if (src.match(media_provider_type_youtube_re)) {
media_provider_type_detected = 'youtube';
} else if (protocolIsKnown(src)) {
// Full URL to a website we know nothing about.
media_provider_type_detected = 'unknown';
}
if (media_provider_type_detected === undefined) {
if (media_provider_type === undefined) {
if (context.media_provider_default) {
// Relative URL, use the default provider if any.
media_provider_type = context.media_provider_default[ast.macro_name];
} else {
media_provider_type = 'local'
}
}
is_url = false;
} else {
if (media_provider_type !== undefined && media_provider_type !== media_provider_type_detected) {
error_message = `detected media provider type "${media_provider_type_detected}", but user also explicitly gave "${media_provider_type}"`;
renderError(context, error_message, ast.args.provider.source_location);
}
if (media_provider_type === undefined) {
media_provider_type = media_provider_type_detected;
}
is_url = true;
}
// Fixup src depending for certain providers.
let relpath_prefix
if (media_provider_type === 'local') {
const path = context.options.ourbigbook_json['media-providers'].local.path;
if (path !== '') {
src = path + URL_SEP + src;
}
} else if (media_provider_type === 'github') {
const github_path = context.options.ourbigbook_json['media-providers'].github.path;
if (
github_path &&
context.options.fs_exists_sync(github_path) &&
!context.options.publish
) {
// Can't join it in here now or else existence check fails.
// But we need to keep this information around to be able to link from inside _out/html/... relative path.
relpath_prefix = path.relative('.', context.options.outdir)
src = `${github_path}/${src}`
} else {
src = `${githubProviderPrefix(context)}/${src}`;
}
}
return {
error_message,
media_provider_type,
is_url,
relpath_prefix,
src,
}
}
function macroImageVideoResolveParamsWithSource(ast, context) {
const ret = macroImageVideoResolveParams(ast, context);
ret.source = context.macros[ast.macro_name].options.source_func(
ast, context, ret.src, ret.media_provider_type, ret.is_url);
return ret;
}
/**
* 'My title', 'My body' => '= My title\n\nMy body'
* 'My title', '{c}\n{tag=My tag}\n\nMy body' => '= My title\n{c}\n{tag=My tag}\n\nMy body'
*/
function modifyEditorInput(title, body) {
let ret = ''
if (title !== undefined) {
ret += `${SHORTHAND_HEADER_CHAR} ${title}\n`
}
let offsetOffset = 0
// Append title to body. Add a newline if the body doesn't start
// with a header argument like `{c}` in:
//
// = h1
// {c}
if (body) {
if (title !== undefined && body[0] !== START_NAMED_ARGUMENT_CHAR) {
ret += '\n'
offsetOffset = 1
}
ret += body
}
return { offset: 1 + offsetOffset, new: ret }
}
exports.modifyEditorInput = modifyEditorInput
const MACRO_IMAGE_VIDEO_OPTIONS = {
captionNumberVisible: function (ast, context) {
return Macro.DESCRIPTION_ARGUMENT_NAME in ast.args ||
macroImageVideoResolveParamsWithSource(ast, context).source !== '';
},
get_title_arg: function(ast, context) {
// Title given explicitly.
if (ast.validation_output[Macro.TITLE_ARGUMENT_NAME].given) {
return ast.args[Macro.TITLE_ARGUMENT_NAME];
}
// Title from src.
const media_provider_type = (macroImageVideoResolveParams(ast, context)).media_provider_type;
if (
ast.validation_output.titleFromSrc.boolean ||
(
!ast.validation_output.titleFromSrc.given &&
context.options.ourbigbook_json['media-providers'][media_provider_type]['title-from-src']
)
) {
let basename_str;
let src = renderArg(ast.args.src, cloneAndSet(context, 'id_conversion', true));
if (media_provider_type === 'local') {
basename_str = urlBasename(src);
} else if (media_provider_type === 'wikimedia') {
basename_str = context.macros[ast.macro_name].options.image_video_basename(src);
} else {
basename_str = src;
}
const title_str = basename_str.replace(/_/g, ' ').replace(/\.[^.]+$/, '') + '.';
return new AstArgument([new PlaintextAstNode(
title_str, ast.source_location)], ast.source_location);
}
// We can't automatically generate one at all.
return undefined;
}
}
const MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS = [
new MacroArgument({
name: 'src',
elide_link_only: true,
mandatory: true,
}),
new MacroArgument({
name: 'alt',
}),
];
const URL_SEP = '/';
exports.URL_SEP = URL_SEP;
const MACRO_WITH_MEDIA_PROVIDER = new Set(['image', 'video']);
const DEFAULT_MACRO_LIST = [
new Macro(
Macro.LINK_MACRO_NAME,
[
new MacroArgument({
name: 'href',
elide_link_only: true,
mandatory: true,
}),
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
cannotContain: new Set([
Macro.LINK_MACRO_NAME,
Macro.X_MACRO_NAME,
'Image',
]),
renderLinksAsPlaintext: true,
}),
],
{
named_args: [
new MacroArgument({
name: 'external',
boolean: true,
}),
new MacroArgument({
name: 'ref',
boolean: true,
}),
],
inline: true,
}
),
new Macro(
Macro.BOLD_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
}),
],
{
inline: true,
}
),
new Macro(
Macro.LINE_BREAK_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
empty: true,
}),
],
{
inline: true,
}
),
new Macro(
// Block code.
Macro.CODE_MACRO_NAME.toUpperCase(),
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
automaticTopicLinks: false,
count_words: true,
ourbigbook_output_prefer_literal: true,
}),
],
{
captionNumberVisible: function (ast, context) {
return Macro.DESCRIPTION_ARGUMENT_NAME in ast.args
},
caption_prefix: 'Code',
id_prefix: 'code',
named_args: [
new MacroArgument({
name: Macro.TITLE_ARGUMENT_NAME,
count_words: true,
}),
new MacroArgument({
name: Macro.DESCRIPTION_ARGUMENT_NAME,
count_words: true,
}),
],
},
),
new Macro(
// Inline code.
Macro.CODE_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
automaticTopicLinks: false,
count_words: true,
ourbigbook_output_prefer_literal: true,
}),
],
{
inline: true,
}
),
new Macro(
Macro.OURBIGBOOK_EXAMPLE_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
ourbigbook_output_prefer_literal: true,
}),
],
{
macro_counts_ignore: function(ast) { return true; }
}
),
new Macro(
'Comment',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
ourbigbook_output_prefer_literal: true,
}),
],
{
macro_counts_ignore: function(ast) { return true; }
}
),
new Macro(
'comment',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
}),
],
{
inline: true,
}
),
new Macro(
Macro.HEADER_MACRO_NAME,
[
new MacroArgument({
name: 'level',
mandatory: true,
positive_nonzero_integer: true,
}),
new MacroArgument({
name: Macro.TITLE_ARGUMENT_NAME,
count_words: true,
cannotContain: new Set([
Macro.LINE_BREAK_MACRO_NAME,
'image',
'br',
]),
renderLinksAsPlaintext: true,
}),
],
{
caption_prefix: 'Section',
default_x_style_full: false,
get_number: function(ast, context) {
let header_tree_node = ast.header_tree_node;
if (header_tree_node === undefined) {
return undefined;
} else {
return header_tree_node.get_nested_number(context.header_tree_top_level);
}
},
show_disambiguate: true,
id_prefix: '',
named_args: [
new MacroArgument({
name: Macro.CAPITALIZE_ARGUMENT_NAME,
boolean: true,
}),
new MacroArgument({
name: Macro.HEADER_CHILD_ARGNAME,
multiple: true,
disabled: true,
}),
new MacroArgument({
name: 'file',
}),
new MacroArgument({
name: 'numbered',
boolean: true,
}),
new MacroArgument({
name: 'parent',
}),
new MacroArgument({
name: 'scope',
boolean: true,
}),
new MacroArgument({
name: 'splitDefault',
boolean: true,
}),
new MacroArgument({
name: 'splitSuffix',
}),
new MacroArgument({
name: 'subdir',
}),
new MacroArgument({
name: Macro.SYNONYM_ARGUMENT_NAME,
boolean: true,
}),
new MacroArgument({
name: 'synonymNoScope',
boolean: true,
}),
new MacroArgument({
name: Macro.HEADER_TAG_ARGNAME,
multiple: true,
}),
new MacroArgument({
name: 'toplevel',
boolean: true,
}),
new MacroArgument({
name: Macro.TITLE2_ARGUMENT_NAME,
multiple: true,
}),
// Should I?
//new MacroArgument({
// name: 'tutorial',
//}),
new MacroArgument({
name: 'wiki',
}),
],
}
),
new Macro(
'Hr',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
empty: true,
}),
]
),
new Macro(
'i',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
}),
],
{
inline: true,
}
),
new Macro(
'Image',
MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS,
Object.assign(
{
caption_prefix: 'Figure',
image_video_content_func: function ({
alt,
ast,
context,
is_url,
media_provider_type,
rendered_attrs,
relpath_prefix,
src,
}) {
let img_html
;({ html: img_html, src } = htmlImg({
alt,
ast,
context,
external: ast.validation_output.external.given ? ast.validation_output.external.boolean : undefined,
media_provider_type,
rendered_attrs,
src,
relpath_prefix,
}))
return img_html
},
named_args: IMAGE_VIDEO_BLOCK_NAMED_ARGUMENTS.concat(IMAGE_INLINE_BLOCK_NAMED_ARGUMENTS),
source_func: function (ast, context, src, media_provider_type, is_url) {
if ('source' in ast.args) {
return renderArg(ast.args.source, cloneAndSet(context, 'html_is_href', true))
} else if (media_provider_type == 'wikimedia') {
return macro_image_video_block_convert_function_wikimedia_source_url +
context.macros[ast.macro_name].options.image_video_basename(src);
} else {
return '';
}
}
},
Object.assign(
{
image_video_basename: function(src) {
return urlBasename(htmlEscapeAttr(src)).replace(
macro_image_video_block_convert_function_wikimedia_source_image_re, '');
},
},
MACRO_IMAGE_VIDEO_OPTIONS,
),
),
),
new Macro(
'image',
MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS,
{
named_args: IMAGE_VIDEO_INLINE_BLOCK_NAMED_ARGUMENTS.concat(IMAGE_INLINE_BLOCK_NAMED_ARGUMENTS),
inline: true,
}
),
new Macro(
Macro.INCLUDE_MACRO_NAME,
[
new MacroArgument({
name: 'href',
mandatory: true,
}),
],
{
macro_counts_ignore: function(ast) { return true; },
named_args: [
new MacroArgument({
name: 'parent',
}),
],
}
),
new Macro(
'JsCanvasDemo',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
mandatory: true,
ourbigbook_output_prefer_literal: true,
}),
],
{
xss_safe: false,
}
),
new Macro(
Macro.LIST_ITEM_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
}),
],
{
auto_parent: Macro.UNORDERED_LIST_MACRO_NAME,
auto_parent_skip: new Set(['Ol']),
}
),
new Macro(
// Block math.
Macro.MATH_MACRO_NAME.toUpperCase(),
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
automaticTopicLinks: false,
count_words: true,
ourbigbook_output_prefer_literal: true,
}),
],
{
caption_prefix: 'Equation',
id_prefix: 'equation',
get_number: function(ast, context) {
// Override because unlike other elements such as images, equations
// always get numbers even if not indexed.
return ast.macro_count;
},
macro_counts_ignore: function(ast) {
return !ast.validation_output.show.boolean;
},
named_args: [
new MacroArgument({
name: Macro.TITLE_ARGUMENT_NAME,
}),
new MacroArgument({
name: Macro.DESCRIPTION_ARGUMENT_NAME,
}),
new MacroArgument({
boolean: true,
default: Macro.BOOLEAN_ARGUMENT_TRUE,
name: 'show',
}),
],
}
),
new Macro(
// Inline math.
Macro.MATH_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
automaticTopicLinks: false,
count_words: true,
ourbigbook_output_prefer_literal: true,
}),
],
{
inline: true,
}
),
new Macro(
'Ol',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
remove_whitespace_children: true,
}),
],
),
new Macro(
Macro.PARAGRAPH_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
}),
],
),
new Macro(
Macro.PLAINTEXT_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
}),
],
{
inline: true,
}
),
new Macro(
capitalizeFirstLetter(Macro.PASSTHROUGH_MACRO_NAME),
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
}),
],
{
xss_safe: false,
}
),
new Macro(
Macro.PASSTHROUGH_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
}),
],
{
inline: true,
xss_safe: false,
}
),
new Macro(
Macro.QUOTE_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
}),
],
{
caption_prefix: 'Quote',
id_prefix: 'quote',
named_args: [
new MacroArgument({
name: Macro.TITLE_ARGUMENT_NAME,
count_words: true,
}),
new MacroArgument({
name: Macro.DESCRIPTION_ARGUMENT_NAME,
count_words: true,
}),
],
}
),
new Macro(
'sub',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
}),
],
{
inline: true,
}
),
new Macro(
'sup',
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
}),
],
{
inline: true,
}
),
new Macro(
Macro.TABLE_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
remove_whitespace_children: true,
count_words: true,
}),
],
{
captionNumberVisible: function (ast, context) {
return Macro.DESCRIPTION_ARGUMENT_NAME in ast.args
},
named_args: [
new MacroArgument({
name: Macro.DESCRIPTION_ARGUMENT_NAME,
count_words: true,
}),
new MacroArgument({
name: Macro.TITLE_ARGUMENT_NAME,
count_words: true,
}),
],
}
),
new Macro(
Macro.TD_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
}),
]
),
new Macro(
Macro.TOPLEVEL_MACRO_NAME,
[
new MacroArgument({
count_words: true,
name: Macro.CONTENT_ARGUMENT_NAME,
}),
],
{
macro_counts_ignore: function(ast) { return true; },
named_args: [
new MacroArgument({
name: Macro.TITLE_ARGUMENT_NAME,
count_words: true,
}),
],
}
),
new Macro(
Macro.TH_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
automaticTopicLinks: false,
count_words: true,
}),
],
),
new Macro(
Macro.TR_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
remove_whitespace_children: true,
}),
],
{
auto_parent: Macro.TABLE_MACRO_NAME,
}
),
new Macro(
Macro.UNORDERED_LIST_MACRO_NAME,
[
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
remove_whitespace_children: true,
}),
],
),
new Macro(
Macro.X_MACRO_NAME,
[
new MacroArgument({
name: 'href',
mandatory: true,
}),
new MacroArgument({
name: Macro.CONTENT_ARGUMENT_NAME,
count_words: true,
cannotContain: new Set([
Macro.LINK_MACRO_NAME,
Macro.X_MACRO_NAME,
]),
renderLinksAsPlaintext: true,
}),
],
{
named_args: [
new MacroArgument({
name: Macro.CAPITALIZE_ARGUMENT_NAME,
boolean: true,
}),
new MacroArgument({
// https://github.com/ourbigbook/ourbigbook/issues/92
name: 'child',
boolean: true,
disabled: true,
}),
new MacroArgument({
name: 'file',
boolean: true,
}),
new MacroArgument({
name: 'full',
boolean: true,
}),
new MacroArgument({
name: 'magic',
boolean: true,
}),
new MacroArgument({
name: 'p',
boolean: true,
}),
new MacroArgument({
// https://github.com/ourbigbook/ourbigbook/issues/92
name: 'parent',
boolean: true,
disabled: true,
}),
new MacroArgument({
name: 'ref',
boolean: true,
}),
new MacroArgument({
name: 'showDisambiguate',
boolean: true,
}),
new MacroArgument({
name: 'topic',
boolean: true,
}),
],
inline: true,
}
),
new Macro(
'Video',
MACRO_IMAGE_VIDEO_POSITIONAL_ARGUMENTS,
Object.assign(
{
caption_prefix: 'Video',
image_video_basename: function(src) {
return urlBasename(htmlEscapeAttr(src)).replace(
macro_image_video_block_convert_function_wikimedia_source_video_re, '$1');
},
image_video_content_func: function ({
alt,
ast,
context,
is_url,
media_provider_type,
rendered_attrs,
src,
}) {
if (media_provider_type === 'youtube') {
let url_start_time;
let video_id;
if (is_url) {
const url = new URL(src);
const url_params = url.searchParams;
if (url_params.has('t')) {
url_start_time = url_params.get('t');
}
if (url.hostname === 'youtube.com' || url.hostname === 'www.youtube.com') {
if (url_params.has('v')) {
video_id = url_params.get('v')
} else {
let message = `youtube URL without video ID "${src}"`;
renderError(context, message, ast.source_location);
return errorMessageInOutput(message, context);
}
} else {
// youtu.be/<ID> and path is "/<ID>" so get rid of "/".
video_id = url.pathname.substring(1);
}
} else {
video_id = src;
}
let start_time;
if ('start' in ast.args) {
start_time = ast.validation_output.start.positive_nonzero_integer;
} else if (url_start_time !== undefined) {
start_time = htmlEscapeAttr(url_start_time);
}
let start;
if (start_time !== undefined) {
start = `?start=${start_time}`;
} else {
start = '';
}
let height
if (ast.validation_output.height.given) {
height = ast.validation_output.height.positive_nonzero_integer
} else {
height = DEFAULT_MEDIA_HEIGHT
}
let width
if (ast.validation_output.width.given) {
width = ast.validation_output.width.positive_nonzero_integer
} else {
const DEFAULT_VIDEO_WIDTH = 560
width = Math.floor(DEFAULT_VIDEO_WIDTH * height / DEFAULT_MEDIA_HEIGHT)
}
return `<div class="float-wrap"><iframe width="${width}" height="${height}" loading="lazy" src="https://www.youtube.com/embed/${htmlEscapeHrefAttr(video_id)}${start}" ` +
`allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></div>`;
} else {
let error
;({ error, href: src } = checkAndUpdateLocalLink({
context,
external: ast.validation_output.external.given ? ast.validation_output.external.boolean : undefined,
hrefNoEscape: src,
media_provider_type,
source_location: ast.args.src.source_location,
}))
let start;
if ('start' in ast.args) {
// https://stackoverflow.com/questions/5981427/start-html5-video-at-a-particular-position-when-loading
start = `#t=${ast.validation_output.start.positive_nonzero_integer}`;
} else {
start = '';
}
return `<video${htmlAttr('src', src + start)}${rendered_attrs} preload="none" controls${alt}></video>${error}`;
}
},
named_args: IMAGE_VIDEO_BLOCK_NAMED_ARGUMENTS.concat(
new MacroArgument({
name: 'start',
positive_nonzero_integer: true,
}),
),
source_func: function (ast, context, src, media_provider_type, is_url) {
if ('source' in ast.args) {
return renderArg(ast.args.source, cloneAndSet(context, 'html_is_href', true))
} else if (media_provider_type === 'youtube') {
if (is_url) {
return htmlEscapeHrefAttr(src);
} else {
return `https://youtube.com/watch?v=${htmlEscapeHrefAttr(src)}`;
}
} else if (media_provider_type === 'wikimedia') {
return macro_image_video_block_convert_function_wikimedia_source_url +
context.macros[ast.macro_name].options.image_video_basename(src);
} else {
return '';
}
}
},
MACRO_IMAGE_VIDEO_OPTIONS,
),
),
];
function createLinkList(context, ast, id, title, target_ids) {
let ret = '';
if (target_ids.size !== 0) {
// TODO factor this out more with real headers.
const target_asts = [];
const idWithPrefix = `${Macro.RESERVED_ID_PREFIX}${id}`
const targetIdsArr = Array.from(target_ids)
ret += htmlToplevelChildModifierById(`<h2 id="${idWithPrefix}"><a href="#${idWithPrefix}">${title} <span class="meta">(${targetIdsArr.length})</span></a></h2>`, idWithPrefix)
let i = 0
for (const target_id of targetIdsArr.sort()) {
let target_ast = context.db_provider.get(target_id, context);
if (
// Possible when user sets an invalid ID on \x with child \x[invalid]{child}.
// The error is caught elsewhere.
target_ast !== undefined
) {
//let counts_str;
//if (target_ast.header_tree_node !== undefined) {
// counts_str = getDescendantCountHtml(target_ast.header_tree_node, false);
//} else {
// counts_str = '';
//}
const xArgs = {
[Macro.CAPITALIZE_ARGUMENT_NAME]: new AstArgument(),
'href': new AstArgument(
[
new PlaintextAstNode(target_id),
],
),
showDisambiguate: new AstArgument(),
}
if (context.options.add_test_instrumentation) {
xArgs[Macro.TEST_DATA_ARGUMENT_NAME] = [new PlaintextAstNode(i.toString())]
}
target_asts.push(new AstNode(
AstType.MACRO,
Macro.LIST_ITEM_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
[
new AstNode( AstType.MACRO, Macro.X_MACRO_NAME, xArgs),
//new AstNode(
// AstType.MACRO,
// Macro.PASSTHROUGH_MACRO_NAME,
// {
// [Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
// [
// new PlaintextAstNode(counts_str),
// ],
// ),
// },
// undefined,
// {
// xss_safe: true,
// }
//),
],
),
},
));
}
i++
}
let ulArgs = {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(target_asts)
}
if (context.options.add_test_instrumentation) {
ulArgs[Macro.TEST_DATA_ARGUMENT_NAME] = [new PlaintextAstNode(id)]
}
const incoming_ul_ast = new AstNode(AstType.MACRO, Macro.UNORDERED_LIST_MACRO_NAME, ulArgs)
const new_context = cloneAndSet(context, 'validateAst', true);
new_context.source_location = ast.source_location;
ret += htmlToplevelChildModifierById(incoming_ul_ast.render(new_context))
}
return ret
}
/** This factors out the wrapper div that every toplevel element must have to do the margin and self link.
*
* It can be called directly for autogenerated elements which might not have a corresponding AST.
*/
function htmlToplevelChildModifierByHref(out, href) {
let linkToSelf
let cls
if (href) {
linkToSelf = `<a${htmlAttr('href', href)}></a>`
cls = ''
} else {
linkToSelf = ''
cls = ' class="nolink"'
}
return out
//return `<div${cls}>${linkToSelf}${out}</div>`
}
function htmlToplevelChildModifierById(out, id) {
const href = id ? `#${id}` : ''
return htmlToplevelChildModifierByHref(out, href)
}
exports.htmlToplevelChildModifierById = htmlToplevelChildModifierById
class OutputFormat {
constructor(id, opts={}) {
this.id = id
this.ext = opts.ext
this.convert_funcs = opts.convert_funcs
if ('toplevelChildModifier' in opts) {
this.toplevelChildModifier = opts.toplevelChildModifier
} else {
this.toplevelChildModifier = (ast, context, out) => out
}
}
}
const OUTPUT_FORMATS_LIST = [
new OutputFormat(
OUTPUT_FORMAT_HTML,
{
ext: HTML_EXT,
toplevelChildModifier: function(ast, context, out) {
let href
if (ast) {
href = xHref(ast, context)
} else {
href = ''
}
return htmlToplevelChildModifierByHref(out, href)
},
convert_funcs: {
[Macro.LINK_MACRO_NAME]: function(ast, context) {
let [href, content, hrefNoEscape] = linkGetHrefAndContent(
ast,
cloneAndSet(context, 'in_a', true),
{ removeProtocol: !context.in_a }
)
if (ast.validation_output.ref.boolean) {
content = `${HTML_REF_MARKER}`;
}
const external = ast.validation_output.external.given ? ast.validation_output.external.boolean : undefined
let attrs = htmlRenderAttrsId(ast, context);
if (context.options.ourbigbook_json.openLinksOnNewTabs) {
attrs += ' target="_blank"'
}
return getLinkHtml({
ast,
attrs,
content,
context,
external,
href,
hrefNoEscape,
source_location: ast.args.href.source_location,
})
},
[Macro.BOLD_MACRO_NAME]: htmlRenderSimpleElem('b'),
[Macro.LINE_BREAK_MACRO_NAME]: function(ast, context) { return '<br>' },
[Macro.CODE_MACRO_NAME.toUpperCase()]: function(ast, context) {
const { title_and_description, multiline_caption } = htmlTitleAndDescription(ast, context, { addTitleDiv: true })
let ret = `<div class="code"${htmlRenderAttrsId(ast, context)}><div${multiline_caption ? ` class="${MULTILINE_CAPTION_CLASS}"` : ''}>`
ret += htmlCode(renderArg(ast.args.content, context))
ret += title_and_description
ret += `</div></div>`
return ret
},
[Macro.CODE_MACRO_NAME]: htmlRenderSimpleElem('code'),
[Macro.OURBIGBOOK_EXAMPLE_MACRO_NAME]: unconvertible,
'Comment': function(ast, context) { return ''; },
'comment': function(ast, context) { return ''; },
[Macro.HEADER_MACRO_NAME]: function(ast, context, opts={}) {
if (opts.extra_returns === undefined) {
opts.extra_returns = {}
}
if (context.in_header) {
// Previously was doing an infinite loop when rendering the parent header.
// But not valid HTML, so I don't think it is worth allowing at all:
// https://stackoverflow.com/questions/17363465/is-nesting-a-h2-tag-inside-another-header-with-h1-tag-semantically-wrong/71130770#71130770
const message = cannotPlaceXInYErrorMessage(Macro.HEADER_MACRO_NAME, Macro.HEADER_MACRO_NAME)
renderError(context, message, ast.source_location);
return errorMessageInOutput(message, context);
}
const context_old = context
context = cloneAndSet(context, 'in_header', true)
const children = ast.args[Macro.HEADER_CHILD_ARGNAME]
const tags = ast.args[Macro.HEADER_TAG_ARGNAME]
if (ast.isSynonym()) {
if (children !== undefined) {
const message = `"synonym" and "child" are incompatible`;
renderError(context, message, children.source_location);
return errorMessageInOutput(message, context);
}
if (tags !== undefined) {
const message = `"synonym" and "tags" are incompatible`;
renderError(context, message, tags.source_location);
return errorMessageInOutput(message, context);
}
return '';
}
let level_int = ast.header_tree_node.get_level();
if (typeof level_int !== 'number') {
throw new Error('header level is not an integer after validation');
}
let custom_args;
const level_int_output = level_int - context.header_tree_top_level + 1;
const is_top_level = level_int === context.header_tree_top_level
let level_int_capped;
if (level_int_output > 6) {
custom_args = {'data-level': new AstArgument([new PlaintextAstNode(
level_int_output.toString(), ast.source_location)], ast.source_location)};
level_int_capped = 6;
} else {
custom_args = {};
level_int_capped = level_int_output;
}
let attrs = htmlRenderAttrs(ast, context, [], custom_args)
let id_attr = htmlRenderAttrsId(ast, context);
let ret = '';
ret += context.renderBeforeNextHeader.map(s => htmlToplevelChildModifierById(s)).join('')
context_old.renderBeforeNextHeader = []
let hasToc = false
if (
level_int !== context.header_tree_top_level ||
context.header_tree.children.length > 1 &&
context.options.render_metadata
) {
let render_toc_ret = renderToc(context)
if (render_toc_ret !== '') {
opts.extra_returns.render_pre = htmlToplevelChildModifierById(render_toc_ret, Macro.TOC_ID)
hasToc = true
}
}
// Div that contains h + on hover span.
let first_header = (
// May fail in some error scenarios.
context.toplevel_ast !== undefined &&
ast.id === context.toplevel_ast.id
)
ret += `<div class="h${first_header ? ' top' : ''}"${id_attr}${hasToc && context.options.add_test_instrumentation ? ' data-has-toc="1"' : ''}>`;
// Self link.
let self_link_context
let self_link_ast
if (context.options.split_headers) {
if (ast.from_include && !context.options.embed_includes) {
self_link_ast = context.db_provider.get(ast.id, context)
self_link_context = context
} else {
self_link_context = cloneAndSet(context, 'to_split_headers', true)
self_link_ast = ast
}
} else {
self_link_context = context
self_link_ast = ast
}
ret += `<div class="notnav">`
if (ast.file) {
ret += `<span class="file" title="This article is about a file." />`
}
ret += `<h${level_int_capped}${attrs}>`
let x_text_options = {
addNumberDiv: true,
show_caption_prefix: false,
style_full: true,
};
const x_text_base_ret = xTextBase(ast, cloneAndSet(context, 'in_a', true), x_text_options);
if (context.toplevel_output_path) {
const rendered_outputs_entry = context.extra_returns.rendered_outputs[context.toplevel_output_path]
if (
// So that when we are rendering h2Render we don't overwrite the "real" output.
!context.skipOutputEntry &&
// Can fail due to splits, which could overwrite nonsplit values.
rendered_outputs_entry !== undefined &&
rendered_outputs_entry.title === undefined
) {
rendered_outputs_entry.title = x_text_base_ret.innerWithDisambiguate
const title_arg = ast.args[Macro.TITLE_ARGUMENT_NAME]
rendered_outputs_entry.titleSource = renderArg(
title_arg,
cloneAndSet(context, 'options',
cloneAndSet(context.options, 'output_format', OUTPUT_FORMAT_OURBIGBOOK)
)
)
rendered_outputs_entry.titleSourceLocation = title_arg.source_location
}
}
let ancestors
if (first_header) {
ancestors = ast.ancestors(context)
if (
// We'll inject this dynamically at runtime. We already do this for the breakcrumb
// so the data is already there. Also full ancestors are not being fetched on render on web,
// only direct parent, so that would need generalizing as well.
!context.options.webMode
) {
for (const ancestor of ancestors.toReversed()) {
if (ancestor.validation_output.scope.given) {
ret += `<a${xHrefAttr(ancestor, context)}>${renderTitlePossibleHomeMarker(ancestor, context)}</a>` +
`<span class="meta"> ${Macro.HEADER_SCOPE_SEPARATOR} </span>`
}
}
}
}
ret += `<a${xHrefAttr(self_link_ast, self_link_context)}>`
ret += x_text_base_ret.full;
ret += `</a>`;
ret += `</h${level_int_capped}>`;
const web_meta = []
if (context.options.h_web_metadata) {
const web_html = `<div class="${H_WEB_CLASS}${first_header ? ' top' : ''}"></div>`
if (first_header) {
web_meta.push(web_html)
} else {
ret += web_html
}
}
// On hover metadata.
let link_to_split;
let parent_links;
{
const items = []
if (
context.options.split_headers &&
context.options.h_show_split_header_link
) {
link_to_split = linkToSplitOpposite(ast, context);
if (link_to_split) {
items.push(`${link_to_split}`);
}
}
let descendant_count = getDescendantCountHtml(context, ast.header_tree_node, { long_style: true });
if (descendant_count !== undefined) {
items.push(`${descendant_count}`);
}
if (!first_header) {
ret += `<span class="hover-meta"> ${items.join('')}</span>`
}
}
// .notnav
ret += '</div>'
// Metadata that shows on separate lines below toplevel header.
let wiki_link;
if (ast.validation_output.wiki.given) {
let wiki = renderArg(ast.args.wiki, context);
if (wiki === '') {
wiki = (renderArg(ast.args[Macro.TITLE_ARGUMENT_NAME], context)).replace(/ /g, '_');
if (ast.validation_output[Macro.DISAMBIGUATE_ARGUMENT_NAME].given) {
wiki += '_(' + (renderArg(ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME], context)).replace(/ /g, '_') + ')'
}
}
wiki_link = `<a href="https://en.wikipedia.org/wiki/${htmlEscapeAttr(wiki)}" class="wiki"></a>`;
}
let ourbigbookLink
if (context.options.ourbigbook_json.web.linkFromStaticHeaderMetaToWeb) {
// Same procedure that can be done from ourbigbook.json to redirect every link out.
const newContext = { ...context }
const newOptions = { ...context.options }
const newOurbigbookJson = { ...context.options.ourbigbook_json }
newContext.options = newOptions
newOptions.ourbigbook_json = newOurbigbookJson
newContext.to_split_headers = true
newOptions.htmlXExtension = false
newOurbigbookJson.xPrefix = ``
let logoPath
if (newContext.options.publish) {
// Relative path.
logoPath = `${newContext.options.template_vars.root_relpath}${newContext.options.logoPath}`
} else {
// Absolute path to local resource.
logoPath = newContext.options.logoPath
}
let p
if (context.options.isindex && ast.is_first_header_in_input_file) {
p = ''
} else {
p = `${URL_SEP}${ast.id}`
}
ourbigbookLink = `<a href="${context.webUrl}${context.options.ourbigbook_json.web.username}${p}">` +
`<img src="${logoPath}" class="logo" alt=""> ${newContext.options.ourbigbook_json.web.hostCapitalized}</a>`;
}
// file handling 1
// Calculate file_link_html
let fileLinkHtml, fileContent
const fileProtocolIsGiven = protocolIsGiven(ast.file)
const renderPostAstsContext = cloneAndSet(context, 'validateAst', true)
renderPostAstsContext.source_location = ast.source_location
if (ast.file) {
if (ast.file.match(media_provider_type_youtube_re)) {
} else {
const readFileRet = context.options.read_file(ast.file, context)
if (readFileRet) {
;({ [Macro.CONTENT_ARGUMENT_NAME]: fileContent } = readFileRet)
}
}
// This section is about.
const pathArg = []
if (fileProtocolIsGiven) {
pathArg.push(
new PlaintextAstNode(' '),
new AstNode(AstType.MACRO,
Macro.LINK_MACRO_NAME,
{
href: new AstArgument([
new PlaintextAstNode(ast.file)
]),
}
),
)
} else {
let curp = ''
pathArg.push(
new PlaintextAstNode(' '),
new AstNode(AstType.MACRO,
// TODO ideally we should link to the corresponding _file entry here
// rather than _raw. We always generate the _file for the file, even in
// non split header mode. To implement this we'd just need to use {extern}
// and calculate the directory offset correctly, much as done for _dir. Lazy now.
// _dir is already linking to _file by default.
Macro.LINK_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
new PlaintextAstNode(FILE_ROOT_PLACEHOLDER)
]),
href: new AstArgument([
new PlaintextAstNode(URL_SEP)
]),
}
),
)
for (const p of ast.file.split(URL_SEP)) {
pathArg.push(new PlaintextAstNode(' ' + URL_SEP + ' '))
if (curp !== '') {
curp += URL_SEP
}
curp += p
const astNodeArgs = {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
new PlaintextAstNode(p)
]),
href: new AstArgument([
new PlaintextAstNode(Macro.HEADER_SCOPE_SEPARATOR + curp)
]),
}
if (context.options.add_test_instrumentation) {
astNodeArgs[Macro.TEST_DATA_ARGUMENT_NAME] = [new PlaintextAstNode(ast.id + Macro.RESERVED_ID_PREFIX + Macro.RESERVED_ID_PREFIX + curp)]
}
pathArg.push(new AstNode(AstType.MACRO, Macro.LINK_MACRO_NAME, astNodeArgs))
}
}
fileLinkHtml = new AstNode(AstType.MACRO, Macro.BOLD_MACRO_NAME, {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(pathArg)
}).render(renderPostAstsContext)
}
// Calculate tag_ids_html
const tag_ids_html_array = [];
let tag_ids_html
const showTags = !ast.from_include || context.options.embed_includes
if (showTags) {
const new_context = cloneAndSet(context, 'validateAst', true);
// This is needed because in case of an an undefined \\x with {parent},
// the undefined target would render as a link on the parent, leading
// to an error that happens on the header, which is before the actual
// root cause.
new_context.ignore_errors = true;
new_context.source_location = ast.source_location;
const target_tag_asts = context.db_provider.get_refs_to_as_asts(
REFS_TABLE_X_CHILD, ast.id, new_context, { current_scope: ast.scope });
for (const target_id of target_tag_asts.map(ast => ast.id).sort()) {
const x_ast = new AstNode(
AstType.MACRO,
Macro.X_MACRO_NAME,
{
[Macro.CAPITALIZE_ARGUMENT_NAME]: new AstArgument(),
'href': new AstArgument(
[
new PlaintextAstNode(target_id),
],
),
},
ast.source_location,
{
scope: ast.scope,
}
);
tag_ids_html_array.push(x_ast.render(new_context));
}
tag_ids_html = `<span class="tags"> Tags: ${tag_ids_html_array.join(', ')}</span>`
}
let toc_link_html;
{
if (
context.forceHeadersHaveTocLink ||
(
!is_top_level &&
checkHasToc(context)
)
) {
let id = ast.id
const fixedScopeRemoval = context.options.fixedScopeRemoval
if (fixedScopeRemoval !== undefined) {
// This is hacky, maybe we shouldn't overload fixedScopeRemoval for this...
// it is used only on web so... lazy to think now.
id = tocId(id.slice(fixedScopeRemoval))
} else {
id = tocIdWithScopeRemoval(id, context)
}
toc_link_html = `<a${htmlAttr(
'href',
'#' + htmlEscapeAttr(id)
)} class="${TOC_LINK_ELEM_CLASS_NAME}"></a>`
}
}
// Calculate header_meta and header_meta
let header_meta = [];
let header_meta_ancestors = [];
let header_meta_file = [];
if (fileLinkHtml !== undefined) {
header_meta_file.push(fileLinkHtml);
}
if (first_header) {
const nAncestors = ancestors.length
if (!context.options.h_web_metadata) {
if (nAncestors) {
const nearestAncestors = ancestors.slice(0, ANCESTORS_MAX).reverse()
const entries = []
for (const ancestor of nearestAncestors) {
entries.push({
href: xHrefAttr(ancestor, context),
content: renderTitlePossibleHomeMarker(ancestor, context),
})
}
// Breadcrumb.
header_meta_ancestors.push(htmlAncestorLinks(
entries,
nAncestors,
{ addTestInstrumentation: context.options.add_test_instrumentation }
))
}
}
} else {
const parent_asts = ast.get_header_parent_asts(context)
parent_links = []
for (const parent_ast of parent_asts) {
// .u for Up
parent_links.push(`<a${xHrefAttr(parent_ast, context)} class="u"> ${renderTitlePossibleHomeMarker(parent_ast, context)}</a>`);
}
parent_links = parent_links.join('');
if (parent_links) {
header_meta.push(parent_links);
}
}
if (first_header) {
if (checkHasToc(context)) {
header_meta.push(`<a${htmlAttr('href', `#${context.options.tocIdPrefix}${Macro.TOC_ID}`)} class="${TOC_LINK_ELEM_CLASS_NAME}"></a>`);
}
} else {
if (toc_link_html) {
header_meta.push(toc_link_html);
}
}
if (wiki_link !== undefined) {
header_meta.push(wiki_link);
}
if (ourbigbookLink !== undefined) {
header_meta.push(ourbigbookLink);
}
if (tag_ids_html_array.length) {
header_meta.push(tag_ids_html);
}
if (first_header) {
if (link_to_split !== undefined) {
header_meta.push(link_to_split);
}
let descendant_count_html = getDescendantCountHtml(context, ast.header_tree_node, { long_style: true });
if (descendant_count_html !== undefined) {
header_meta.push(descendant_count_html);
}
}
const metas = [
[web_meta, ''],
[header_meta_ancestors, 'ancestors'],
[header_meta, ''],
[header_meta_file, 'file'],
]
const header_has_meta = metas.some((m) => m[0].length > 0)
if (header_has_meta) {
ret += `<nav class="h-nav h-nav-toplevel">`;
}
if (context.options.h_web_ancestors && first_header && !context.options.isindex) {
ret += `<div class="nav ${H_ANCESTORS_CLASS}"></div>`
}
let i = 0
for (const [meta, cls] of metas) {
if (meta.length > 0) {
ret += `<div class="nav${cls ? ` ${cls}` : ''}">${meta.join('')}</div>`;
}
i++
}
if (header_has_meta) {
ret += `</nav>`;
}
ret += `</div>`;
if (showTags) {
if (children !== undefined) {
ret += headerCheckChildTagExists(ast, context, children, 'child')
}
if (tags !== undefined) {
ret += headerCheckChildTagExists(ast, context, tags, 'tag')
}
}
// Variables we want permanently modify the context.
context_old.toc_was_rendered = context.toc_was_rendered
// file handling 2
const renderPostAsts = []
if (ast.file) {
const absPref = fileProtocolIsGiven ? '' : URL_SEP
const hasVideoExt = VIDEO_EXTENSIONS.has(pathSplitext(ast.file)[1])
if (IMAGE_EXTENSIONS.has(pathSplitext(ast.file)[1])) {
renderPostAsts.push(new AstNode(
AstType.MACRO,
'Image',
{
'src': new AstArgument([ new PlaintextAstNode(absPref + ast.file) ]),
// For now we force provider to be local, thus preventing {file} to be about a provider
// which feels like a reasonable restriction. If we wanted to remove this, we'd need to add
// a provider argument to \H, which falls into the wider question of allowing every \Image
// option to also be forwrded through \H.
'provider': new AstArgument([ new PlaintextAstNode('local') ]),
},
))
} else if (
hasVideoExt ||
ast.file.match(media_provider_type_youtube_re)
) {
const args = {
'src': new AstArgument([ new PlaintextAstNode(absPref + ast.file) ]),
}
if (hasVideoExt) {
args.provider = new AstArgument([ new PlaintextAstNode('local') ])
}
renderPostAsts.push(new AstNode(
AstType.MACRO,
'Video',
args
))
} else {
// Plaintext file. Possibly embed into HTML.
const protocol = protocolGet(ast.file)
if (protocol === null || protocol === 'file') {
if (fileContent !== undefined) {
// https://stackoverflow.com/questions/1677644/detect-non-printable-characters-in-javascript
const bold_file_ast = new AstNode(AstType.MACRO, Macro.BOLD_MACRO_NAME, {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
new PlaintextAstNode(`${ast.file}`),
])
})
let no_preview_msg
if (/[\x00]/.test(fileContent)) {
no_preview_msg = ` it is a binary file (contains \\x00) of unsupported type (e.g. not an image).`
} else if (
fileContent.length > FILE_PREVIEW_MAX_SIZE &&
!context.in_split_headers &&
!context.options.hFileShowLarge
) {
no_preview_msg = `it is too large (> ${FILE_PREVIEW_MAX_SIZE} bytes)`
}
if (no_preview_msg) {
context_old.renderBeforeNextHeader.push(new AstNode(AstType.MACRO, Macro.PARAGRAPH_MACRO_NAME, {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
bold_file_ast,
new PlaintextAstNode(` was not rendered because ${no_preview_msg}`),
])
}).render(renderPostAstsContext))
} else {
context_old.renderBeforeNextHeader.push(new AstNode(AstType.MACRO, Macro.PARAGRAPH_MACRO_NAME, {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
bold_file_ast
])
}).render(renderPostAstsContext))
context_old.renderBeforeNextHeader.push(new AstNode(
AstType.MACRO,
Macro.CODE_MACRO_NAME.toUpperCase(), {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([ new PlaintextAstNode(fileContent)]),
},
).render(renderPostAstsContext))
}
}
}
}
}
if (renderPostAsts.length) {
opts.extra_returns.render_post = renderPostAsts.map(a => htmlToplevelChildModifierById(a.render(renderPostAstsContext))).join('')
}
// Tagged list for non-toplevel headers.
if (
level_int !== context.header_tree_top_level &&
!ast.from_include
) {
const tagged = Array.from(context.db_provider.get_refs_to_as_ids(REFS_TABLE_X_CHILD, ast.id, true)).sort()
if (tagged.length) {
context_old.renderBeforeNextHeader.push(htmlToplevelChildModifierById(
new AstNode(
AstType.MACRO,
Macro.PARAGRAPH_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
new AstNode(
AstType.MACRO,
Macro.BOLD_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
new AstNode(
AstType.MACRO,
Macro.PASSTHROUGH_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
new PlaintextAstNode(TAGS_MARKER)
]),
}
),
new PlaintextAstNode(` Tagged`),
])
}
),
new AstNode(
AstType.MACRO,
Macro.UNORDERED_LIST_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
tagged.map(t => {
const arg = {
[Macro.CAPITALIZE_ARGUMENT_NAME]: new AstArgument(),
'href': new AstArgument([ new PlaintextAstNode(t) ]),
}
if (context.options.add_test_instrumentation) {
arg[Macro.TEST_DATA_ARGUMENT_NAME] = new AstArgument([
new PlaintextAstNode(`tagged-not-toplevel_${ast.id}_${t}`)
])
}
return new AstNode(
AstType.MACRO,
Macro.LIST_ITEM_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument([
new AstNode(AstType.MACRO, Macro.X_MACRO_NAME, arg)
])
}
)
})
)
}
),
]),
}
).render(renderPostAstsContext)
))
}
}
return ret;
},
'Hr': function(ast, context) { return '<div><hr></div>' },
'i': htmlRenderSimpleElem('i'),
'Image': macroImageVideoBlockConvertFunction,
'image': function(ast, context) {
let alt_arg;
if (ast.args.alt === undefined) {
alt_arg = ast.args.src;
} else {
alt_arg = ast.args.alt;
}
let alt = htmlAttr('alt', htmlEscapeAttr(renderArg(alt_arg, context)));
let rendered_attrs = htmlRenderAttrsId(ast, context, ['height', 'width']);
let { error_message, media_provider_type, src } = macroImageVideoResolveParams(ast, context);
const external = ast.validation_output.external.given ? ast.validation_output.external.boolean : undefined
let { html: imgHtml } = htmlImg({ alt, ast, context, external, inline: true, media_provider_type, rendered_attrs, src })
if (error_message) {
imgHtml += errorMessageInOutput(error_message, context)
}
return imgHtml
},
[Macro.INCLUDE_MACRO_NAME]: unconvertible,
'JsCanvasDemo': function(ast, context) {
return htmlCode(
renderArg(ast.args.content, context),
{ 'class': 'ourbigbook-js-canvas-demo' }
);
},
[Macro.LIST_ITEM_MACRO_NAME]: htmlRenderSimpleElem('li'),
[Macro.MATH_MACRO_NAME.toUpperCase()]: function(ast, context) {
let katex_output = htmlKatexConvert(ast, context)
let ret = ``
if (ast.validation_output.show.boolean) {
const { href, multiline_caption, title_and_description } = htmlTitleAndDescription(ast, context, { addTitleDiv: true })
ret += `<div class="math"${htmlRenderAttrsId(ast, context)}>`
ret += `<div${multiline_caption ? ` class="${MULTILINE_CAPTION_CLASS}"` : ''}>`
ret += `<div class="equation">`
ret += `<div>${katex_output}</div>`
ret += `<div class="number"><a${href}>(${context.macros[ast.macro_name].options.get_number(ast, context)})</a></div>`
ret += `</div>`
ret += title_and_description
ret += `</div>`
ret += `</div>`
}
return ret
},
[Macro.MATH_MACRO_NAME]: function(ast, context) {
// KaTeX already adds a <span> for us.
return htmlKatexConvert(ast, context);
},
'Ol': htmlRenderSimpleElem('ol', UL_OL_OPTS),
[Macro.PARAGRAPH_MACRO_NAME]: htmlRenderSimpleElem(
// Paragraph. Can't be p because p can only contain phrasing it is shorthand:
// https://stackoverflow.com/questions/7168723/unordered-list-in-a-paragraph-element
'div',
{
attrs: { 'class': 'p' },
}
),
[Macro.PLAINTEXT_MACRO_NAME]: function(ast, context) {
const text = ast.text
if (ast.automaticTopicLinks) {
let ret = ''
const plaintextSplitPunct = text.split(AUTOMATIC_TOPIC_LINK_PUNCTUATION_REGEX_CAPTURE)
for (const [i, p] of plaintextSplitPunct.entries()) {
if (i % 2 === 0) {
// Add the preceeding separator part. After this one,
// all others will be id text + separator pairs (except possibly the last id text).
let j = 0
while (
j < p.length &&
titleToIdContext(p[j], undefined, context) === ''
) {
j++
}
ret += p.slice(0, j)
// Calculate idComponents and textComponents.
// The dog is blue.
// ['the', 'dog', 'is', 'blue']
let idComponents = []
// ['The', ' ', 'dog', ' ', 'is', ' ', 'blue', '.']
let textComponents = []
while (j < p.length) {
let start = j
while (
j < p.length &&
titleToIdContext(p[j], undefined, context) !== ''
) {
j++
}
let curComp = p.slice(start, j)
textComponents.push(curComp)
idComponents.push(titleToIdContext(curComp, undefined, context))
start = j
while (
j < p.length &&
titleToIdContext(p[j], undefined, context) === ''
) {
j++
}
if (start < j) {
textComponents.push(p.slice(start, j))
}
}
// Now using idComponents and textComponents and the previously fetched
// topic IDs, do the final text calculation.
let idComponentsI = 0
let textComponentsI = 0
while (idComponentsI < idComponents.length) {
let k
for (k = context.options.automaticTopicLinksMaxWords; k > 0; k--) {
let topicIdNoInflection = idComponents.slice(idComponentsI, idComponentsI + k).join(ID_SEPARATOR)
let topicId
if (context.automaticTopicLinkIds.has(topicIdNoInflection)) {
topicId = topicIdNoInflection
} else {
const singular = pluralizeWrap(topicIdNoInflection, 1)
if (
singular !== topicIdNoInflection &&
context.automaticTopicLinkIds.has(singular)
) {
topicId = singular
}
}
if (topicId) {
const textComponentsAdvance = k * 2 - 1
let xContent = textComponents.slice(textComponentsI, textComponentsI + textComponentsAdvance).join('')
ret += `<a href="${htmlEscapeHrefAttr(`/${WEB_TOPIC_PATH}/${topicId}`)}" class="t">` +
`${htmlEscapeContent(xContent)}` +
`</a>`
// Topic found, wrap it into a topic link.
idComponentsI += k
textComponentsI += textComponentsAdvance
break
}
}
// Topic not found, add the first word and separator as is
// and move on to the next word.
if (k === 0) {
ret += textComponents[textComponentsI]
textComponentsI++
idComponentsI++
j++
}
// Add the separator after that last considered text.
if (textComponentsI < textComponents.length) {
ret += textComponents[textComponentsI]
textComponentsI++
}
}
} else {
// Punctuation. Add as is.
ret += p
}
}
return ret
} else {
return htmlEscapeContext(context, text)
}
},
[capitalizeFirstLetter(Macro.PASSTHROUGH_MACRO_NAME)]: function(ast, context) {
return `<div class="float-wrap">${renderArgNoescape(ast.args.content, context)}</div>`
},
[Macro.PASSTHROUGH_MACRO_NAME]: function(ast, context) {
return renderArgNoescape(ast.args.content, context)
},
[Macro.QUOTE_MACRO_NAME]: htmlRenderSimpleElem('blockquote', { wrap: true }),
'sub': htmlRenderSimpleElem('sub'),
'sup': htmlRenderSimpleElem('sup'),
[Macro.TABLE_MACRO_NAME]: function(ast, context) {
let attrs = htmlRenderAttrsId(ast, context);
let content = renderArg(ast.args.content, context);
let ret = ``;
let { description, force_separator, multiline_caption } = getDescription(ast.args.description, context)
ret += `<div class="table"${attrs}><div${multiline_caption ? ` class="${MULTILINE_CAPTION_CLASS}"` : ''}>`;
// TODO not using caption because I don't know how to allow the caption to be wider than the table.
// I don't want the caption to wrap to a small table size.
//
// If we ever solve that, re-add the following style:
//
// caption {
// color: black;
// text-align: left;
// }
//
//Caption on top as per: https://tex.stackexchange.com/questions/3243/why-should-a-table-caption-be-placed-above-the-table */
let href = htmlAttr('href', '#' + htmlEscapeAttr(ast.id));
if (ast.index_id || ast.validation_output.description.given) {
const { full: title, inner, innerNoDiv } = xTextBase(ast, context, {
addTitleDiv: true,
href_prefix: href,
force_separator,
})
const title_and_description = getTitleAndDescription({ title, description, inner, innerNoDiv })
ret += `<div class="caption">${title_and_description}</div>`;
}
ret += `<table>${content}</table>`;
ret += `</div></div>`;
return ret;
},
[Macro.TD_MACRO_NAME]: htmlRenderSimpleElem('td'),
[Macro.TOPLEVEL_MACRO_NAME]: function(ast, context) {
let title = ast.args[Macro.TITLE_ARGUMENT_NAME];
if (title === undefined) {
let text_title;
if (Macro.TITLE_ARGUMENT_NAME in context.options) {
text_title = context.options[Macro.TITLE_ARGUMENT_NAME];
} else if (context.header_tree.children.length > 0) {
text_title = renderArg(
context.header_tree.children[0].ast.args[Macro.TITLE_ARGUMENT_NAME],
cloneAndSet(context, 'id_conversion', true)
);
} else {
text_title = 'dummy title because title is mandatory in HTML';
}
title = new AstArgument([
new PlaintextAstNode(text_title, ast.source_location)],
ast.source_location,
text_title
);
}
let body = renderArg(ast.args.content, context);
// Footer metadata.
if (context.options.render_metadata) {
body += renderToc(context)
}
body += context.renderBeforeNextHeader.map(s => htmlToplevelChildModifierById(s)).join('')
const ancestors = context.toplevel_ast.ancestors(context)
if (
context.toplevel_ast !== undefined &&
context.options.render_metadata
) {
{
const target_ids = context.db_provider.get_refs_to_as_ids(
REFS_TABLE_X_CHILD, context.toplevel_ast.id, true);
body += createLinkList(context, ast, TAGGED_ID_UNRESERVED, `${TAGS_MARKER} Tagged`, target_ids)
}
// Ancestors
{
if (ancestors.length !== 0) {
// TODO factor this out more with real headers.
body += htmlToplevelChildModifierById(`<h2 id="${ANCESTORS_ID}"><a href="#${ANCESTORS_ID}">${HTML_PARENT_MARKER} Ancestors <span class="meta">(${ancestors.length})</span></a></h2>`, ANCESTORS_ID)
const ancestor_id_asts = [];
let i = 0
for (const ancestor of ancestors) {
//let counts_str;
//if (ancestor.header_tree_node !== undefined) {
// counts_str = getDescendantCountHtml(ancestor.header_tree_node, false);
//} else {
// counts_str = '';
//}
const xArgs = {
[Macro.CAPITALIZE_ARGUMENT_NAME]: new AstArgument(),
'href': new AstArgument(
[
new PlaintextAstNode(ancestor.id),
],
),
showDisambiguate: new AstArgument(),
}
if (context.options.add_test_instrumentation) {
xArgs[Macro.TEST_DATA_ARGUMENT_NAME] = [new PlaintextAstNode(i.toString())]
}
ancestor_id_asts.push(new AstNode(
AstType.MACRO,
Macro.LIST_ITEM_MACRO_NAME,
{
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
[
new AstNode(AstType.MACRO, Macro.X_MACRO_NAME, xArgs),
//new AstNode(
// AstType.MACRO,
// Macro.PASSTHROUGH_MACRO_NAME,
// {
// [Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(
// [
// new PlaintextAstNode(counts_str),
// ],
// ),
// },
// undefined,
// {
// xss_safe: true,
// }
//),
],
),
},
));
i++
}
const ulArgs = {
[Macro.CONTENT_ARGUMENT_NAME]: new AstArgument(ancestor_id_asts)
}
if (context.options.add_test_instrumentation) {
ulArgs[Macro.TEST_DATA_ARGUMENT_NAME] = [new PlaintextAstNode(ANCESTORS_ID_UNRESERVED)]
}
const incoming_ul_ast = new AstNode(
AstType.MACRO,
'Ol',
ulArgs,
);
const new_context = cloneAndSet(context, 'validateAst', true);
new_context.source_location = ast.source_location;
body += htmlToplevelChildModifierById(incoming_ul_ast.render(new_context))
}
}
{
const target_ids = context.db_provider.get_refs_to_as_ids(REFS_TABLE_X, context.toplevel_ast.id);
body += createLinkList(context, ast, INCOMING_LINKS_ID_UNRESERVED, `${INCOMING_LINKS_MARKER} Incoming links`, target_ids)
}
{
const target_ids = context.db_provider.get_refs_to_as_ids(REFS_TABLE_SYNONYM, context.toplevel_ast.id);
body += createLinkList(context, ast, SYNONYM_LINKS_ID_UNRESERVED, `${SYNONYM_LINKS_MARKER} Synonyms`, target_ids)
}
}
let ret
if (context.options.body_only) {
ret = body;
} else {
let template;
if (context.options.template !== undefined) {
template = context.options.template;
} else {
template = `<!doctype html>
<html lang=en>
<head>
<meta charset=utf-8>
<title>{{ title }}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>{{ style }}</style>
{{ head }}</head>
<body class="ourbigbook">
{{ body }}
{{ post_body }}</body>
</html>
`;
}
let root_page;
if (context.options.htmlXExtension) {
context.options.template_vars.html_ext = '.html';
context.options.template_vars.html_index = '/index.html';
root_page = context.options.template_vars.root_relpath + INDEX_BASENAME_NOEXT + '.' + HTML_EXT;
} else {
context.options.template_vars.html_ext = '';
context.options.template_vars.html_index = '';
if (context.options.template_vars.root_relpath === '') {
root_page = '.'
} else {
root_page = context.options.template_vars.root_relpath;
}
}
if (root_page === context.toplevel_output_path) {
root_page = '';
}
const render_env = {
body,
root_page,
title:
ancestors.toReversed().map(a =>
a.validation_output.scope.given ?
renderArg(a.args[Macro.TITLE_ARGUMENT_NAME], context) + ` ${Macro.HEADER_SCOPE_SEPARATOR} ` :
''
).join('') +
renderArg(title, context)
,
};
if (context.options.auto_generated_source) {
render_env.input_path = ''
} else {
render_env.input_path = context.options.input_path
}
const github_prefix = githubProviderPrefix(context)
if (github_prefix) {
render_env.github_prefix = github_prefix
}
Object.assign(render_env, context.options.template_vars);
// Resolve relative styles and scripts.
let relative_scripts = [];
for (const script of context.options.template_scripts_relative) {
relative_scripts.push(`<script src="${context.options.template_vars.root_relpath}${script}"></script>`);
}
const toplevel_scope = context.toplevel_ast ? context.toplevel_ast.calculate_scope() : undefined
const ourbigbook_redirect_prefix_raw = toplevel_scope ? `${toplevel_scope}${URL_SEP}` : ''
const ourbigbook_redirect_prefix = JSON.stringify(ourbigbook_redirect_prefix_raw).replace(/</g, '\\u003c')
const data_script = `<script>
window.ourbigbook_split_headers = ${context.options.split_headers};
window.ourbigbook_html_x_extension = ${context.options.htmlXExtension};
window.ourbigbook_redirect_prefix = ${ourbigbook_redirect_prefix};
</script>
`
render_env.post_body = data_script + relative_scripts.join('') + render_env.post_body + "<script>ourbigbook_runtime.ourbigbook_runtime()</script>";
let relative_styles = [];
for (const style of context.options.template_styles_relative) {
relative_styles.push(`@import "${context.options.template_vars.root_relpath}${style}";\n`);
}
render_env.style = relative_styles.join('') + render_env.style
render_env.is_index_article = !!(context.options.isindex && context.toplevel_ast.is_first_header_in_input_file && !toplevel_scope)
const { Liquid } = require('liquidjs');
ret = (new Liquid()).parseAndRenderSync(
template,
render_env,
{
strictFilters: true,
strictVariables: true,
}
);
}
return ret;
},
[Macro.TH_MACRO_NAME]: htmlRenderSimpleElem('th'),
[Macro.TR_MACRO_NAME]: function(ast, context) {
let content_ast = ast.args.content;
let content = renderArg(content_ast, context);
let res = '';
if (ast.args.content.get(0).macro_name === Macro.TH_MACRO_NAME) {
if (
ast.parent_argument_index === 0 ||
ast.parent_argument.get(ast.parent_argument_index - 1).args.content.get(0).macro_name !== Macro.TH_MACRO_NAME
) {
res += `<thead>`;
}
}
if (ast.args.content.get(0).macro_name === Macro.TD_MACRO_NAME) {
if (
ast.parent_argument_index === 0 ||
ast.parent_argument.get(ast.parent_argument_index - 1).args.content.get(0).macro_name !== Macro.TD_MACRO_NAME
) {
res += `<tbody>`;
}
}
res += `<tr${htmlRenderAttrsId(ast, context)}>${content}</tr>`;
if (ast.args.content.get(0).macro_name === Macro.TH_MACRO_NAME) {
if (
ast.parent_argument_index === ast.parent_argument.length() - 1 ||
ast.parent_argument.get(ast.parent_argument_index + 1).args.content.get(0).macro_name !== Macro.TH_MACRO_NAME
) {
res += `</thead>`;
}
}
if (ast.args.content.get(0).macro_name === Macro.TD_MACRO_NAME) {
if (
ast.parent_argument_index === ast.parent_argument.length() - 1 ||
ast.parent_argument.get(ast.parent_argument_index + 1).args.content.get(0).macro_name !== Macro.TD_MACRO_NAME
) {
res += `</tbody>`;
}
}
return res;
},
[Macro.UNORDERED_LIST_MACRO_NAME]: htmlRenderSimpleElem('ul', UL_OL_OPTS),
[Macro.X_MACRO_NAME]: function(ast, context) {
let [href, content, target_ast] = xGetHrefContent(
ast,
context,
{ showDisambiguate: ast.validation_output.showDisambiguate.boolean }
)
let incompatible_pair
if (ast.validation_output.full.given) {
if (ast.validation_output.ref.given) {
incompatible_pair = ['full', 'ref']
}
if (ast.validation_output.content.given) {
incompatible_pair = ['full', Macro.CONTENT_ARGUMENT_NAME]
}
if (ast.validation_output.c.given) {
incompatible_pair = ['full', 'c']
}
if (ast.validation_output.p.given) {
incompatible_pair = ['full', 'p']
}
} else if (ast.validation_output.content.given) {
if (ast.validation_output.ref.given) {
incompatible_pair = [Macro.CONTENT_ARGUMENT_NAME, 'ref']
}
if (ast.validation_output.c.given) {
incompatible_pair = [Macro.CONTENT_ARGUMENT_NAME, 'c']
}
if (ast.validation_output.p.given) {
incompatible_pair = [Macro.CONTENT_ARGUMENT_NAME, 'p']
}
} else if (ast.validation_output.ref.given) {
if (ast.validation_output.c.given) {
incompatible_pair = ['ref', 'c']
}
if (ast.validation_output.p.given) {
incompatible_pair = ['ref', 'p']
}
}
if (incompatible_pair) {
const message = `"${incompatible_pair[0]}" and "${incompatible_pair[1]}" are incompatible`;
renderError(context, message, ast.source_location);
content = errorMessageInOutput(message, context);
} else if (ast.validation_output.ref.boolean) {
content = HTML_REF_MARKER;
}
if (!context.in_a) {
// Counts.
let counts_str;
if (
// Happens on error case of linking to non existent ID.
target_ast === undefined ||
// Happens for cross links. TODO make those work too...
target_ast.parent_ast === undefined
) {
counts_str = '';
} else {
const counts = getDescendantCount(target_ast.header_tree_node);
for (let i = 0; i < counts.length; i++) {
counts[i] = formatNumberApprox(counts[i]);
}
counts_str = `\nword count: ${counts[0]}\ndescendant word count: ${counts[2]}\ndescendant count: ${counts[1]}`;
}
const attrs = htmlRenderAttrsId(ast, context);
// It would be cleaner to pass this up from xHrefParts. But lazy.
let target = ''
if (context.options.ourbigbook_json.openLinksOnNewTabs && href) {
const splitFragment = href.split('#')
let href_path, fragment
if (splitFragment.length > 1) {
;[href_path, fragment] = splitFragment
} else {
href_path = href
}
if (href_path !== '') {
target = ' target="_blank"'
}
}
return `<a${href}${attrs}` +
(context.options.internalLinkMetadata ? htmlAttr('title', 'internal link' + counts_str) : '') +
target +
getTestData(ast, context) +
`>${content}</a>`
} else {
return content
}
},
'Video': macroImageVideoBlockConvertFunction,
[Macro.TEST_SANE_ONLY]: htmlRenderSimpleElem('div'),
[decapitalizeFirstLetter(Macro.TEST_SANE_ONLY)]: htmlRenderSimpleElem('span'),
},
}
),
new OutputFormat(
OUTPUT_FORMAT_ID,
{
ext: 'id',
convert_funcs: {
[Macro.LINK_MACRO_NAME]: function(ast, context) {
const [href, content, hrefNoEscape] = linkGetHrefAndContent(ast, context);
return content;
},
[Macro.BOLD_MACRO_NAME]: idConvertSimpleElem(),
[Macro.LINE_BREAK_MACRO_NAME]: function(ast, context) { return '\n'; },
[Macro.CODE_MACRO_NAME.toUpperCase()]: idConvertSimpleElem(),
[Macro.CODE_MACRO_NAME]: idConvertSimpleElem(),
[Macro.OURBIGBOOK_EXAMPLE_MACRO_NAME]: unconvertible,
'Comment': function(ast, context) { return ''; },
'comment': function(ast, context) { return ''; },
[Macro.HEADER_MACRO_NAME]: idConvertSimpleElem(),
'Hr': function(ast, context) { return '\n'; },
'i': idConvertSimpleElem(),
'Image': function(ast, context) { return ''; },
'image': function(ast, context) { return ''; },
[Macro.INCLUDE_MACRO_NAME]: unconvertible,
'JsCanvasDemo': idConvertSimpleElem(),
[Macro.LIST_ITEM_MACRO_NAME]: idConvertSimpleElem(),
[Macro.MATH_MACRO_NAME.toUpperCase()]: idConvertSimpleElem(),
[Macro.MATH_MACRO_NAME]: idConvertSimpleElem(),
'Ol': idConvertSimpleElem(),
[Macro.PARAGRAPH_MACRO_NAME]: idConvertSimpleElem(),
[Macro.PLAINTEXT_MACRO_NAME]: function(ast, context) { return ast.text },
[capitalizeFirstLetter(Macro.PASSTHROUGH_MACRO_NAME)]: idConvertSimpleElem(),
[Macro.PASSTHROUGH_MACRO_NAME]: idConvertSimpleElem(),
[Macro.QUOTE_MACRO_NAME]: idConvertSimpleElem(),
'sub': idConvertSimpleElem(),
'sup': idConvertSimpleElem(),
[Macro.TABLE_MACRO_NAME]: idConvertSimpleElem(),
[Macro.TD_MACRO_NAME]: idConvertSimpleElem(),
[Macro.TOPLEVEL_MACRO_NAME]: idConvertSimpleElem(),
[Macro.TH_MACRO_NAME]: idConvertSimpleElem(),
[Macro.TR_MACRO_NAME]: idConvertSimpleElem(),
[Macro.UNORDERED_LIST_MACRO_NAME]: idConvertSimpleElem(),
[Macro.X_MACRO_NAME]: function(ast, context) {
if (ast.args.content) {
return idConvertSimpleElem(Macro.CONTENT_ARGUMENT_NAME)(ast, context)
} else {
return idConvertSimpleElem('href')(ast, context)
}
},
'Video': function(ast, context) { return ''; },
[Macro.TEST_SANE_ONLY]: idConvertSimpleElem(),
[decapitalizeFirstLetter(Macro.TEST_SANE_ONLY)]: idConvertSimpleElem(),
}
}
),
]
function ourbigbookCodeMathInline(c) {
return function(ast, context) {
const content = renderArg(ast.args.content, cloneAndSet(context, 'in_literal', true))
if (content.indexOf(c) === -1) {
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, context))
if (dolog) {
console.log(`ourbigbookCodeMathInline: newline=${require('util').inspect(newline)}`)
}
return `${newline}${c}${content}${c}`
} else {
return ourbigbookConvertSimpleElem(ast, context)
}
}
}
function ourbigbookCodeMathBlock(c) {
return function(ast, context) {
const contextInLiteral = cloneAndSet(context, 'in_literal', true)
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, contextInLiteral))
const content = renderArg(ast.args.content, contextInLiteral)
let delim = c + c
while (content.indexOf(delim) !== -1) {
delim += c
}
const attrs = ourbigbookConvertArgs(ast, context, { skip: new Set([Macro.CONTENT_ARGUMENT_NAME]) }).join('')
return `${newline}${delim}
${content}
${delim}${attrs === '' ? '' : '\n'}${attrs}`
}
}
/** Escape all ourbigbook constructs that must be escaped, except
* for those that only need to be escaped if they are at the start of a line. */
function ourbigbookEscapeNotStart(text) {
return text.replace(MUST_ESCAPE_CHARS_REGEX_CHAR_CLASS_REGEX, `${ESCAPE_CHAR}$1`)
}
exports.ourbigbookEscapeNotStart = ourbigbookEscapeNotStart
function ourbigbookEscape(text) {
return ourbigbookEscapeNotStart(text).replace(MUST_ESCAPE_CHARS_AT_START_REGEX_CHAR_CLASS_REGEX, '$1\\$2')
}
exports.ourbigbookEscape = ourbigbookEscape
/** Get the preferred x href for the ourbigbook output format of an \x. */
function ourbigbookGetXHref({
ast,
c,
context,
file,
for_header_parent,
href,
magic,
p,
target_ast,
target_id,
}) {
const hrefOrig = href
if (!file) {
href = href.replaceAll(ID_SEPARATOR, ' ')
const href_plural = pluralizeWrap(href, 2)
if (p) {
let target_ast_plural = context.db_provider.get(magicTitleToId(href_plural), context)
if (
// When we have \x without magic to a destination that exists
// in both plural and singular, we can't use the magic plural,
// or it will resolve to plural rather than the correct singular.
!magic &&
target_ast &&
target_ast_plural
) {
return { override_href: `<${href}>${ourbigbookConvertArgs(ast, context, { skip: new Set(['c', 'href', 'magic']) }).join('')}` }
}
href = href_plural
if (target_ast_plural) {
target_ast = target_ast_plural
}
}
if (!target_ast) {
return { override_href: `${ourbigbookConvertSimpleElem(ast, context)} ${renderErrorXUndefined(ast, context, target_id)}` }
}
let was_magic_plural, was_magic_uppercase
if (magic && !for_header_parent) {
if (
!target_ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME] &&
href === href_plural
) {
was_magic_plural = true
}
const components = href.split(Macro.HEADER_SCOPE_SEPARATOR)
const c = components[components.length - 1][0]
was_magic_uppercase = c && (c === c.toUpperCase())
}
const href_from_id = href
if (!(
(
target_ast.macro_name === Macro.HEADER_MACRO_NAME &&
target_ast.validation_output.file.given
)
)) {
const macro = context.macros[target_ast.macro_name];
const title_arg = macro.options.get_title_arg(target_ast, context);
href = renderArg(title_arg, cloneAndSet(context, 'id_conversion', true));
href = href.replaceAll(Macro.HEADER_SCOPE_SEPARATOR, ' ')
let was_pluralized
let disambiguate
const disambiguate_arg = target_ast.args[Macro.DISAMBIGUATE_ARGUMENT_NAME];
if (disambiguate_arg) {
disambiguate = renderArg(disambiguate_arg, cloneAndSet(context, 'id_conversion', true))
} else {
disambiguate = ''
}
const explicit_id = target_ast.first_toplevel_child || (
target_ast.validation_output[Macro.ID_ARGUMENT_NAME] &&
target_ast.validation_output[Macro.ID_ARGUMENT_NAME].given
)
if (macro.options.id_prefix) {
href = `${macro.options.id_prefix} ${href}`
} else {
const first_ast = title_arg.get(0);
if (!(
(
target_ast.macro_name === Macro.HEADER_MACRO_NAME &&
target_ast.validation_output.c.boolean
) ||
(
first_ast &&
first_ast.node_type !== AstType.PLAINTEXT
) ||
for_header_parent
)) {
if (href.length) {
if (c || was_magic_uppercase) {
href = href[0].toUpperCase() + href.substring(1)
} else {
href = href[0].toLowerCase() + href.substring(1)
}
}
}
if (
was_magic_plural ||
p
) {
const words = hrefOrig.split(PLURALIZE_WORD_SPLIT_REGEX)
const lastWordOrig = words[words.length - 1]
let href_plural
if (magic) {
href_plural = forceLastWordReplace(href, lastWordOrig)
} else {
href_plural = pluralizeWrap(href, 2)
}
let disambiguate_sep
if (disambiguate) {
disambiguate_sep = ID_SEPARATOR + disambiguate
} else {
disambiguate_sep = ''
}
let target_scope = target_ast.scope
if (target_scope) {
target_scope = `${target_scope}${Macro.HEADER_SCOPE_SEPARATOR}`
} else {
target_scope = ''
}
const plural_id = `${target_scope}${magicTitleToId(href_plural, context)}${disambiguate_sep}`
const plural_target = context.db_provider.get(plural_id, context)
if (!plural_target || plural_target.id !== target_ast.id) {
const singular_id = `${target_scope}${magicTitleToId(pluralizeWrap(href, 1), context)}${disambiguate_sep}`
const singular_target = context.db_provider.get(singular_id, context)
if ((!singular_target || singular_target.id !== target_ast.id) && !explicit_id) {
// This can happen due to pluralize bugs:
// https://github.com/plurals/pluralize/issues/172
// Just bail out in those cases.
return { override_href: ourbigbookConvertSimpleElem(ast, context) }
}
}
href = href_plural
was_pluralized = true
}
}
if (explicit_id && magicTitleToId(href, context) !== target_ast.id) {
href = href_from_id
} else {
if (disambiguate_arg) {
if (was_pluralized) {
// TODO https://github.com/ourbigbook/ourbigbook/issues/244
return { override_href: ourbigbookConvertSimpleElem(ast, context) }
}
href = `${href} (${disambiguate})`;
}
let target_scope = target_ast.scope
if (target_scope) {
let scopeSrc = hrefOrig.split(Macro.HEADER_SCOPE_SEPARATOR).slice(0, -1).join(Macro.HEADER_SCOPE_SEPARATOR)
if (scopeSrc) {
scopeSrc += Macro.HEADER_SCOPE_SEPARATOR
}
href = `${scopeSrc}${href}`
}
}
if (isAbsoluteXref(target_id, context)) {
href = target_id[0] + href
}
}
}
return { href, override_href: undefined }
}
function ourbigbookLi(marker, opts={}) {
let { oneNewlineMax } = opts
if (oneNewlineMax === undefined) {
oneNewlineMax = true
}
return function(ast, context) {
if (!ast.args.content || Object.keys(ast.args).length !== 1) {
return ourbigbookConvertSimpleElem(ast, context)
} else {
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, context, { oneNewlineMax }))
if (dolog) {
console.log(`ourbigbookLi: newline=${require('util').inspect(newline)}`)
}
const content = renderArg(ast.args.content, context)
const content_indent = content.replace(/\n(.)/g, '\n $1')
let marker_eff
if (!content_indent) {
marker_eff = marker.substring(0, marker.length - 1)
} else {
marker_eff = marker
}
return `${newline}${marker_eff}${content_indent}`
}
}
}
/** How many newlines to add before each macro.
* This is by far the hardest part of -O bigb output to shorthand macros!
* Countless hours have spent on this function.
*
* @return {Number}
*/
function ourbigbookNewlinesBefore(ast, context, options={}) {
const { oneNewlineMax } = options
if (ast.parent_argument_index === 0) {
return 0
} else {
if (
context.macros[ast.macro_name].inline
//(
// // It is a bit sad that we have to use this "am I on toplevel" checks processing here.
// // It would be saner if indented blocks were exactly the same as toplevel.
// // But it just intuitively feels that the "no loose on toplevel, but loose in subelements"
// // sound good, so going for it like that now... I think this is implemented by always adding
// // a PARAGRAPH Token on toplevel in case we ever want to dump that.
// //ast.parent_ast.macro_name === Macro.TOPLEVEL_MACRO_NAME ||
// (
// //!(
// // ast.parent_ast.macro_name === Macro.PARAGRAPH_MACRO_NAME &&
// // ast.parent_ast.parent_ast.macro_name === Macro.TOPLEVEL_MACRO_NAME
// //) &&
// ast.parent_ast.macro_name !== Macro.TOPLEVEL_MACRO_NAME &&
// (
// ( ast.parent_argument.has_paragraph ) ||
// ( !ast.parent_argument.has_paragraph && !ast.parent_argument.not_all_block )
// )
// )
//)
) {
const prevMacro = ast.parent_argument.get(ast.parent_argument_index - 1)
if (prevMacro && !context.macros[prevMacro.macro_name].inline) {
return 1
}
} else {
let n
if (ast.parent_ast.macro_name === Macro.PARAGRAPH_MACRO_NAME) {
// We are bb or dd below (direct descendant of paragraph but not first sibling.)
//
// \TestSaneOnly[
// \TestSaneOnly[aa]
// \TestSaneOnly[bb]
//
// \TestSaneOnly[cc]
// \TestSaneOnly[dd]
// ]
n = 1
} else {
const prevMacro = ast.parent_argument.get(ast.parent_argument_index - 1)
if (
prevMacro && context.macros[prevMacro.macro_name].inline ||
oneNewlineMax
) {
n = 1
} else {
n = 2
}
if (context.last_render) {
if (context.last_render[context.last_render.length - 1] === '\n') {
n--
if (n && context.last_render[context.last_render.length - 2] === '\n') {
n--
}
}
}
}
return n
}
}
}
function ourbigbookUl(ast, context) {
if (!ast.args.content || Object.keys(ast.args).length !== 1) {
return ourbigbookConvertSimpleElem(ast, context)
} else {
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, context))
const argstr = renderArg(ast.args.content, context)
if (dolog) {
console.log(`ourbigbookUl: newline=${require('util').inspect(newline)}`)
}
return `${newline}${argstr}`
}
}
function ourbigbookPreferLiteral(ast, context, ast_arg, arg, open, close) {
let rendered_arg
let delim_repeat
let has_newline
if (
// Can only be literal if there is just one single plaintext node.
ast_arg.asts.length === 1 &&
ast_arg.asts[0].node_type === AstType.PLAINTEXT
) {
const macro = context.macros[ast.macro_name]
const macroArg = macro.name_to_arg[arg.name]
const text = ast_arg.asts[0].text
if (
// Prefer literals if any escapes would be needed.
arg.ourbigbook_output_prefer_literal
// In the past we had more complicated rules here that would consider
// how many escapes wre needed. But I've been feeling that I never want
// literals except for specific arguments where I always want a literal.
//|| (
// ourbigbookEscape(text) !== text &&
// !(
// macroArg.elide_link_only &&
// protocolIsKnown(text) &&
// !hasShorthandLinkEndChars(text)
// )
//)
) {
rendered_arg = ast_arg.asts[0].text
delim_repeat = 2
while (
rendered_arg.indexOf(open.repeat(delim_repeat)) !== -1 ||
rendered_arg.indexOf(close.repeat(delim_repeat)) !== -1
) {
delim_repeat++
}
has_newline = rendered_arg.indexOf('\n') !== -1
if (
rendered_arg[0] === open &&
!has_newline
) {
rendered_arg = ESCAPE_CHAR + rendered_arg
}
if (
rendered_arg[rendered_arg.length - 1] === close &&
!has_newline
) {
rendered_arg = rendered_arg.substring(0, rendered_arg.length - 1) + ESCAPE_CHAR + close
}
}
}
if (!rendered_arg) {
// Not a literal.
delim_repeat = 1
rendered_arg = renderArg(ast_arg, context)
has_newline = rendered_arg.indexOf('\n') !== -1
}
return { delim_repeat, has_newline, rendered_arg }
}
function ourbigbookConvertArgs(ast, context, options={}) {
let { onelineArg } = options
const ret = options.ret || []
const skip = options.skip || new Set()
const modify_callbacks = options.modify_callbacks || {}
const macro = context.macros[ast.macro_name]
if (onelineArg === undefined) {
onelineArg = false
}
let named_args = Macro.COMMON_ARGNAMES.concat(macro.options.named_args.map(arg => arg.name)).filter(
(argname) => !skip.has(argname) && ast.validation_output[argname].given
)
const ret_args = []
for (const arg of macro.positional_args) {
const ret_arg = []
const argname = arg.name
if (!skip.has(argname) && ast.validation_output[argname].given) {
const astArg = ast.args[argname]
let hasBlockChild = false
for (const ast of astArg.asts) {
if (!context.macros[ast.macro_name].inline) {
hasBlockChild = true
break
}
}
let { delim_repeat, has_newline, rendered_arg } = ourbigbookPreferLiteral(
ast, context, astArg, arg, START_POSITIONAL_ARGUMENT_CHAR, END_POSITIONAL_ARGUMENT_CHAR)
if (argname in modify_callbacks) {
rendered_arg = modify_callbacks[argname](ast, context, rendered_arg)
}
ret_arg.push(START_POSITIONAL_ARGUMENT_CHAR.repeat(delim_repeat))
if (arg.remove_whitespace_children) {
ret_arg.push('\n')
} else {
if (has_newline || hasBlockChild) {
if (rendered_arg[0] !== '\n') {
ret_arg.push('\n')
}
}
}
ret_arg.push(rendered_arg)
if (
(has_newline || hasBlockChild) &&
(
rendered_arg.length === 0 ||
rendered_arg[rendered_arg.length - 1] !== '\n'
)
) {
ret_arg.push('\n')
}
ret_arg.push(END_POSITIONAL_ARGUMENT_CHAR.repeat(delim_repeat))
}
if (ret_arg.length) {
ret_args.push(ret_arg)
}
}
const orderedNamedArgs = []
for (const argname of [
Macro.TITLE_ARGUMENT_NAME,
Macro.ID_ARGUMENT_NAME,
Macro.DISAMBIGUATE_ARGUMENT_NAME,
]) {
const idx = named_args.indexOf(argname)
if (idx !== -1) {
orderedNamedArgs.push(argname)
named_args.splice(idx, 1)
}
}
orderedNamedArgs.push(...named_args.sort())
for (const argname of orderedNamedArgs) {
const arg = macro.named_args[argname]
const validation_output = ast.validation_output[argname]
let ast_args
if (arg.multiple) {
ast_args = ast.args[argname].asts.map(ast => ast.args.content)
} else {
ast_args = [ast.args[argname]]
}
for (const ast_arg of ast_args) {
const ret_arg = []
const macro_arg = macro.name_to_arg[argname]
let hasBlockChild = false
for (const ast of ast_arg.asts) {
if (!context.macros[ast.macro_name].inline) {
hasBlockChild = true
break
}
}
let { delim_repeat, has_newline, rendered_arg } = ourbigbookPreferLiteral(
ast, context, ast_arg, arg, START_NAMED_ARGUMENT_CHAR, END_NAMED_ARGUMENT_CHAR)
if (argname in modify_callbacks) {
rendered_arg = modify_callbacks[argname](ast, context, ast_arg, rendered_arg)
}
let skip_val = false
if (macro_arg.boolean) {
const argstr_default = macro_arg.default === undefined ? Macro.BOOLEAN_ARGUMENT_FALSE : Macro.BOOLEAN_ARGUMENT_TRUE
const argstr_eff = validation_output.boolean ? Macro.BOOLEAN_ARGUMENT_TRUE : Macro.BOOLEAN_ARGUMENT_FALSE
if (argstr_default === argstr_eff) {
continue
}
skip_val = validation_output.boolean
} else if(rendered_arg === '') {
skip_val = true
}
ret_arg.push(
START_NAMED_ARGUMENT_CHAR.repeat(delim_repeat) +
argname
)
if (!skip_val) {
ret_arg.push(NAMED_ARGUMENT_EQUAL_CHAR)
if ((has_newline || hasBlockChild) && rendered_arg[0] !== '\n') {
ret_arg.push('\n')
}
ret_arg.push(rendered_arg)
if (
(has_newline || hasBlockChild) &&
(
rendered_arg.length === 0 ||
rendered_arg[rendered_arg.length - 1] !== '\n'
)
) {
ret_arg.push('\n')
}
}
ret_arg.push(END_NAMED_ARGUMENT_CHAR.repeat(delim_repeat))
if (ret_arg.length) {
ret_args.push(ret_arg)
}
}
}
let i = 0
for (const ret_arg of ret_args) {
ret.push(...ret_arg)
if (!macro.inline && i !== ret_args.length - 1 && !onelineArg) {
// Add newline between last positional and first named argument.
ret.push('\n')
}
i++
}
return ret
}
function ourbigbookConvertSimpleElem(ast, context, opts={}) {
const ret = []
ret.push(ESCAPE_CHAR + ast.macro_name)
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, context))
if (dolog) {
console.log(`ourbigbookConvertSimpleElem ${ast.macro_name} newline=${require('util').inspect(newline)}`)
}
ourbigbookConvertArgs(ast, context, { ret, ...opts })
return newline + ret.join('')
}
OUTPUT_FORMATS_LIST.push(
new OutputFormat(
// This is hard, especially for shorthand constructs and in particular newline placement.
// A general principle is: each macro optionally outputs newlines only before itself,
// and never after.
OUTPUT_FORMAT_OURBIGBOOK,
{
ext: OURBIGBOOK_EXT,
convert_funcs: {
[Macro.LINK_MACRO_NAME]: function(ast, context) {
const href = renderArg(ast.args.href, context)
if (protocolIsKnown(href)) {
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, context))
if (dolog && newline) {
console.log(`Macro.LINK_MACRO_NAME newline: ${require('util').inspect(newline, { depth: null })}`)
}
return `${newline}${href}${ourbigbookConvertArgs(ast, context, { skip: new Set(['href']) }).join('')}`
} else {
return ourbigbookConvertSimpleElem(ast, context)
}
},
[Macro.BOLD_MACRO_NAME]: ourbigbookConvertSimpleElem,
[Macro.LINE_BREAK_MACRO_NAME]: function(ast, context) {
const nextMacro = ast.parent_argument.get(ast.parent_argument_index + 1)
const prevAst = ast.parent_argument.get(ast.parent_argument_index - 1)
if (
ast.parent_argument_index === 0 ||
!context.macros[ast.parent_argument.get(ast.parent_argument_index - 1).macro_name].inline ||
// Last child in argument, need to be explicit or else ignored by newline removal.
!nextMacro ||
!context.macros[nextMacro.macro_name].inline ||
// Multiple brs together, can't render them all as \n or else we get a paragraph,
(prevAst && prevAst.macro_name === Macro.LINE_BREAK_MACRO_NAME)
) {
return ourbigbookConvertSimpleElem(ast, context)
} else {
return '\n'
}
},
[Macro.CODE_MACRO_NAME.toUpperCase()]: ourbigbookCodeMathBlock(SHORTHAND_CODE_CHAR),
[Macro.CODE_MACRO_NAME]: ourbigbookCodeMathInline(SHORTHAND_CODE_CHAR),
[Macro.OURBIGBOOK_EXAMPLE_MACRO_NAME]: ourbigbookConvertSimpleElem,
'Comment': ourbigbookConvertSimpleElem,
'comment': ourbigbookConvertSimpleElem,
[Macro.HEADER_MACRO_NAME]: function(ast, context) {
function modifyCallback(ast, context, arg, rendered_arg) {
const { target_ast, target_id } = xGetTargetAstBase({
context,
do_magic_title_to_id: true,
do_singularize: false,
scope: ast.scope,
target_id: rendered_arg,
})
if (!target_ast) {
return `${rendered_arg} ${renderErrorXUndefined(ast, context, rendered_arg, { source_location: arg.source_location })}`
}
const href = ourbigbookGetXHref({
ast,
context,
href: rendered_arg,
target_id,
target_ast,
c: false,
p: false,
magic: true,
for_header_parent: true,
}).href
// Return only some whitelisted characters to prevent creating new elements
// or breaking out of parent=} argument. Maybe one day we can force parent=
// to be always literal. But this would require changing the tokenizer somehow,
// not sure it would be super easy.
return href.replace(MUST_ESCAPE_CHARS_REGEX_CHAR_CLASS_REGEX, ' ').replace(MUST_ESCAPE_CHARS_AT_START_REGEX_CHAR_CLASS_REGEX, '$1').replace(/ +/g, ' ').replace(/^ | $/g, '')
}
let level = ast.header_tree_node.get_level() - context.header_tree_top_level + 1;
let output_level
if (
ast.validation_output.parent.given ||
ast.isSynonym()
) {
output_level = 1
} else {
output_level = level
}
const skip = new Set(['level', 'title'])
if (level === 1) {
// Happens on split headers.
skip.add('parent')
}
const args_string = ourbigbookConvertArgs(
ast,
context,
{
skip,
modify_callbacks: {
'child': modifyCallback,
'parent': modifyCallback,
'tag': modifyCallback,
}
}
).join('')
const titleRender = renderArg(ast.args.title, context)
if (titleRender.indexOf('\n') > -1) {
return ourbigbookConvertSimpleElem(ast, context)
}
return `${ast.parent_argument_index === 0 ? '' : '\n\n'}${SHORTHAND_HEADER_CHAR.repeat(output_level)} ${titleRender}${args_string ? '\n' : '' }${args_string}`
},
'Hr': ourbigbookConvertSimpleElem,
'i': ourbigbookConvertSimpleElem,
'Image': ourbigbookConvertSimpleElem,
'image': ourbigbookConvertSimpleElem,
[Macro.INCLUDE_MACRO_NAME]: function (ast, context) {
return ourbigbookConvertSimpleElem(ast, context, { onelineArg: true })
},
'JsCanvasDemo': ourbigbookConvertSimpleElem,
[Macro.LIST_ITEM_MACRO_NAME]: ourbigbookLi(SHORTHAND_LIST_START),
[Macro.MATH_MACRO_NAME.toUpperCase()]: ourbigbookCodeMathBlock(SHORTHAND_MATH_CHAR),
[Macro.MATH_MACRO_NAME]: ourbigbookCodeMathInline(SHORTHAND_MATH_CHAR),
'Ol': ourbigbookConvertSimpleElem,
[Macro.PARAGRAPH_MACRO_NAME]: function(ast, context) {
if (!ast.args.content || Object.keys(ast.args).length !== 1) {
return ourbigbookConvertSimpleElem(ast, context)
} else {
const rendered_arg = renderArg(ast.args.content, context)
let newline
if (ast.parent_argument_index === 0) {
newline = ''
} else {
// TODO this is horrible. We should actually try and properly calculate
// and normalize newlines of the other elements such that this is either
// always '\n' or always '\n\n'.
if (context.last_render[context.last_render.length - 1] === '\n') {
newline = '\n'
} else {
newline = '\n\n'
}
}
if (dolog && newline) {
console.log(`Macro.PARAGRAPH_MACRO_NAME newline: ${require('util').inspect(newline, { depth: null })}`)
}
return `${newline}${rendered_arg}`
}
},
[Macro.PLAINTEXT_MACRO_NAME]: function(ast, context) {
const text = ast.text
if (context.in_literal) {
return text
}
// Elide link only does not need escaping.
const parentArg = ast.parent_argument
if (
// There is just one string literal.
parentArg.asts.length === 1 &&
parentArg.asts[0].node_type === AstType.PLAINTEXT
) {
const parentAst = ast.parent_ast
if (parentAst) {
const macro = context.macros[parentAst.macro_name]
const macroArg = macro.name_to_arg[parentArg.argument_name]
if (
macroArg.elide_link_only &&
protocolIsKnown(text) &&
!hasShorthandLinkEndChars(text)
) {
return text
}
}
}
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, context))
if (dolog) {
console.log(`Macro.PLAINTEXT_MACRO_NAME: newline=${require('util').inspect(newline)} text=${require('util').inspect(text)}`)
}
return newline + ourbigbookEscape(text)
},
[capitalizeFirstLetter(Macro.PASSTHROUGH_MACRO_NAME)]: ourbigbookConvertSimpleElem,
[Macro.PASSTHROUGH_MACRO_NAME]: ourbigbookConvertSimpleElem,
[Macro.QUOTE_MACRO_NAME]: ourbigbookLi(SHORTHAND_QUOTE_START, { oneNewlineMax: false }),
'sub': ourbigbookConvertSimpleElem,
'sup': ourbigbookConvertSimpleElem,
[Macro.TABLE_MACRO_NAME]: ourbigbookUl,
[Macro.TD_MACRO_NAME]: ourbigbookLi(SHORTHAND_TD_START),
[Macro.TOPLEVEL_MACRO_NAME]: function(ast, context) {
let ret = renderArg(ast.args[Macro.CONTENT_ARGUMENT_NAME], context)
let newline = ''
// TODO a bit horrible but works.
if (ret[ret.length - 1] !== '\n') {
newline = '\n'
}
if (dolog && newline) {
console.log(`${Macro.TOPLEVEL_MACRO_NAME}: newline=${require('util').inspect(newline)} ast.macro_name=${ast.macro_name}`)
}
return ret + newline
},
[Macro.TH_MACRO_NAME]: ourbigbookLi(SHORTHAND_TH_START),
[Macro.TR_MACRO_NAME]: ourbigbookUl,
[Macro.UNORDERED_LIST_MACRO_NAME]: ourbigbookUl,
[Macro.X_MACRO_NAME]: function(ast, context) {
const hrefArg = ast.args.href
let href = renderArg(hrefArg, context)
let hasLinkEndChar = false
const newline = '\n'.repeat(ourbigbookNewlinesBefore(ast, context))
if (ast.validation_output.topic.boolean) {
for (const c of href) {
if (SHORTHAND_LINK_END_CHARS.has(c)) {
hasLinkEndChar = true
break
}
}
const nextAst = ast.parent_argument.get(ast.parent_argument_index + 1)
let nextAstRender
if (nextAst) {
nextAstRender = nextAst.render(context)
}
if (dolog && newline) {
console.log(`Macro.X_MACRO_NAME newline: ${require('util').inspect(newline, { depth: null })}`)
}
if (
!hasLinkEndChar &&
// Check that what immediately follows does not start with a character
// that would glue to the link e.g.
// <#ab>cd
// cannot be:
// #abcd
!(nextAstRender && !SHORTHAND_LINK_END_CHARS.has(nextAstRender[0]))
) {
return `${newline}${SHORTHAND_TOPIC_CHAR}${href}`
} else {
return `${newline}${SHORTHAND_X_START}${SHORTHAND_TOPIC_CHAR}${href}${SHORTHAND_X_END}`
}
}
//if (AstType.PLAINTEXT === ast.args.href[0] === SHORTHAND_TOPIC_CHAR) {
// return `<${href}>`
//}
// Remove any > from the ref. There's currently no way to escape them, would cut argument short.
let { target_id, target_ast } = xGetTargetAst(ast, context)
const magic = ast.validation_output.magic.boolean
if (!magic && href !== magicTitleToId(href, context)) {
// Explicit IDs with weird characters weird cannot be converted to shorthand, e.g.
//
// = Dollar
// {{id=$}}
//
// \x[[$]]
return ourbigbookConvertSimpleElem(ast, context)
}
href = renderArg(hrefArg, cloneAndSet(context, 'in_literal', true))
let override_href
;({ href, override_href } = ourbigbookGetXHref({
ast,
c: ast.validation_output.c.boolean,
context,
file: ast.validation_output.file.given,
href,
magic,
p: ast.validation_output.p.boolean,
target_ast,
target_id,
}))
if (override_href) {
return override_href
}
href = href.replace(/[ >]+/g, ' ')
if (dolog && newline) {
console.log(`Macro.X_MACRO_NAME newline: ${require('util').inspect(newline, { depth: null })}`)
}
return `${newline}${SHORTHAND_X_START}${href}${SHORTHAND_X_END}` +
ourbigbookConvertArgs(ast, context, { skip: new Set(['c', 'href', 'magic', 'p']) }).join('')
},
'Video': ourbigbookConvertSimpleElem,
[Macro.TEST_SANE_ONLY]: ourbigbookConvertSimpleElem,
[decapitalizeFirstLetter(Macro.TEST_SANE_ONLY)]: ourbigbookConvertSimpleElem,
}
}
)
)
const OUTPUT_FORMATS = {}
exports.OUTPUT_FORMATS = OUTPUT_FORMATS
for (const output_format of OUTPUT_FORMATS_LIST) {
OUTPUT_FORMATS[output_format.id] = output_format
}