OurBigBook Docs
ourbigbook
#!/usr/bin/env node
const now = performance.now.bind(performance)
const china_dictatorship = require('china-dictatorship');
if (!china_dictatorship.get_data().includes("Tiannmen Square protests")) throw 0;
const child_process = require('child_process')
const fs = require('fs')
const path = require('path')
const os = require('os')
const isConversionWorker = process.argv[2] === '--internal-conversion-worker' && !!process.send
let workerData, grantDatabaseWrite
// This library is terrible, too much magic, hard to understand interface,
// does not do some obvious basics.
const commander = require('commander');
const is_installed_globally = require('is-installed-globally');
const readCb = require('read');
const { Liquid } = require('liquidjs');
const lodash = require('lodash')
const { DataTypes, Op, Sequelize } = require('sequelize')
const ourbigbook = require('ourbigbook');
const {
cssEscapeDoubleQuotedString,
DIR_PREFIX,
FILE_PREFIX,
FILE_TYPE_FILE,
FILE_TYPE_DIRECTORY,
INDEX_BASENAME_NOEXT,
OURBIGBOOK_EXT,
RAW_PREFIX,
URL_SEP,
} = ourbigbook
const ourbigbook_nodejs = require('ourbigbook/nodejs');
const ourbigbook_nodejs_front = require('ourbigbook/nodejs_front');
const ourbigbook_nodejs_webpack_safe = require('ourbigbook/nodejs_webpack_safe');
const { cliInt } = ourbigbook_nodejs_webpack_safe
const {
ARTICLE_HASH_LIMIT_MAX,
ARTICLE_RENDER_BATCH_LIMIT,
articleHash,
hashToHex,
read_include,
WebApi,
} = require('ourbigbook/web_api');
const DEFAULT_TEMPLATE_BASENAME = 'ourbigbook.liquid.html';
const OURBIGBOOK_TEX_BASENAME = 'ourbigbook.tex';
const LOG_OPTIONS = new Set([
'ast',
'ast-simple',
'db',
'headers',
'tokens',
]);
const SASS_EXT = '.scss';
const DEFAULT_IGNORE_BASENAMES = [
ourbigbook.RESERVED_ID_SEPARATOR,
'.git',
ourbigbook_nodejs_webpack_safe.TMP_DIRNAME,
'_raw',
'_dir',
];
const DEFAULT_IGNORE_BASENAMES_SET = new Set(DEFAULT_IGNORE_BASENAMES);
const MESSAGE_PREFIX_EXTRACT_IDS = 'extract_ids'
const MESSAGE_PREFIX_RENDER = 'render'
const MESSAGE_SKIP_BY_TIMESTAMP = `skipped by timestamp`
const WEB_MAX_RETRIES = 3
const WEB_RETRY_DELAY_MS = 1000
// A bounded pool reused across the extraction and rendering barriers. Each
// worker runs one file at a time; the coordinator serializes its transactions.
// Processes also isolate native SQLite bindings from concurrent JS runtimes.
class CliWorkerPool {
constructor(filename, jobs, workerData) {
this.filename = filename
this.jobs = jobs
this.workerData = workerData
this.workers = []
this.writeQueue = []
}
addWorker() {
const worker = child_process.fork(this.filename, ['--internal-conversion-worker'], {
serialization: 'advanced',
stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
})
const slot = { worker }
slot.ready = new Promise((resolve, reject) => { slot.startup = { resolve, reject } })
slot.exited = new Promise(resolve => {
worker.on('exit', code => {
if (!this.closing && !this.error) {
this.fail(new Error(`conversion worker exited unexpectedly (status ${code})`))
}
resolve()
})
})
worker.on('error', error => this.fail(error))
worker.on('message', result => {
if (this.error) return
if (result.ready) {
slot.startup.resolve()
slot.startup = undefined
return
}
if (result.acquireWrite) {
this.writeQueue.push(slot)
this.grantWrite()
return
}
if (result.releaseWrite) {
this.writer = undefined
this.grantWrite()
return
}
const pending = slot.pending
slot.pending = undefined
if (result.error) {
this.fail(new Error(result.error))
if (pending) pending.reject(this.error)
} else if (pending) {
pending.resolve(result)
}
})
this.workers.push(slot)
worker.send(this.workerData)
}
grantWrite() {
if (!this.writer && this.writeQueue.length && !this.error) {
this.writer = this.writeQueue.shift()
this.writer.worker.send({ writeGranted: true })
}
}
fail(error) {
if (this.error) return
this.error = error
for (const slot of this.workers) {
if (slot.startup) slot.startup.reject(error)
if (slot.pending) {
slot.pending.reject(error)
slot.pending = undefined
}
// A crashed worker may have held the write lock. Stop its peers rather
// than leaving them waiting forever; SQLite rolls back open transactions.
slot.worker.kill()
}
}
async run(tasks) {
if (this.error) throw this.error
while (this.workers.length < Math.min(this.jobs, tasks.length)) this.addWorker()
const results = new Array(tasks.length)
let next = 0
let hadError = false
await Promise.all(this.workers.map(async slot => {
await slot.ready
while (!hadError && !this.error && next < tasks.length) {
const i = next++
const result = await new Promise((resolve, reject) => {
slot.pending = { resolve, reject }
slot.worker.send(tasks[i])
})
results[i] = result
if (result.had_error) hadError = true
}
}))
return results.filter(result => result !== undefined)
}
async close() {
this.closing = true
for (const slot of this.workers) {
if (!this.error) slot.worker.send({ close: true })
}
await Promise.all(this.workers.map(slot => slot.exited))
if (this.error) throw this.error
}
}
class DbProviderDbAdapter {
constructor(nonOurbigbookOptions) {
}
}
function addUsername(idNoUsername, username) {
if (idNoUsername !== null && idNoUsername !== undefined) {
if (idNoUsername === '') {
return `${ourbigbook.AT_MENTION_CHAR}${username}`
} else {
return `${ourbigbook.AT_MENTION_CHAR}${username}/${idNoUsername}`
}
}
return null
}
function assertApiStatus(status, data) {
//console.log(require('child_process').execSync(`printf 'count '; sqlite3 /home/ciro/bak/git/ourbigbook/web/db.sqlite3 "select to_id_index,from_id,to_id,defined_at from Ref where from_id = '@barack-obama' and type = 0 order by to_id_index" | wc -l`).toString())
//console.log(require('child_process').execSync(`sqlite3 /home/ciro/bak/git/ourbigbook/web/db.sqlite3 "select to_id_index,from_id,to_id,defined_at from Ref where from_id = '@barack-obama' and type = 0 order by to_id_index"`).toString())
if (status !== 200) {
console.error(`HTTP error status: ${status}`);
console.error(`Error messages from server:`);
const errors = data.errors
if (errors instanceof Array) {
for (const error of data.errors) {
console.error(error);
}
} else {
if (errors === undefined) {
console.error(data);
} else {
console.error(errors);
}
}
cli_error()
}
}
async function read(opts) {
return new Promise((resolve, reject) => {
// TODO allow program to exit on Ctrl + C, currently ony cancels read
// https://stackoverflow.com/questions/24037545/how-to-hide-password-in-the-nodejs-console
readCb(opts, (err, line) => {
resolve([err, line])
})
})
}
// Like read, but:
// * Ctrl + C works and quits program
// * no password support
async function readStdin(opts) {
const chunks = [];
for await (const chunk of process.stdin) chunks.push(chunk);
return Buffer.concat(chunks).toString('utf8');
}
// Reconcile the database with information that depends only on existence of Ourbigbook files, notably:
// - remove any IDs from deleted files https://github.com/ourbigbook/ourbigbook/issues/125
async function reconcile_db_and_filesystem(input_path, ourbigbook_options, nonOurbigbookOptions) {
const sequelize = nonOurbigbookOptions.sequelize
if (sequelize) {
const newNonOurbigbookOptions = ourbigbook.cloneAndSet(
nonOurbigbookOptions, 'ourbigbook_paths_converted_only', true)
await convert_directory(
input_path,
ourbigbook_options,
newNonOurbigbookOptions,
);
const inputRelpath = path.relative(nonOurbigbookOptions.ourbigbook_json_dir, input_path)
let pathPrefix
if (inputRelpath) {
pathPrefix = inputRelpath + path.sep
} else {
pathPrefix = ''
}
pathPrefix += '%'
const { File, Id, Ref, Render } = sequelize.models
const ourbigbook_paths_converted = newNonOurbigbookOptions.ourbigbook_paths_converted
const [,,,file_rows] = await Promise.all([
// Delete IDs from deleted files.
// It is possible in pure SQL, but not supported in sequelize:
// https://cirosantilli.com/delete-with-join-sql
// so we do a double query for now.
sequelize.models.Id.findAll({
attributes: ['id'],
include: [
{
model: File,
as: 'idDefinedAt',
where: {
path: {
[Op.not]: ourbigbook_paths_converted,
[Op.like]: pathPrefix,
},
},
attributes: [],
},
],
}).then(ids => Id.destroy({ where: { id: ids.map(id => id.id ) } })),
// Delete Refs from deleted files.
Ref.findAll({
attributes: ['id'],
include: [
{
model: File,
as: 'definedAt',
where: {
path: {
[Op.not]: ourbigbook_paths_converted,
[Op.like]: pathPrefix,
}
},
attributes: [],
},
],
}).then(ids => Ref.destroy({ where: { id: ids.map(id => id.id ) } })),
// Delete deleted Files
File.destroy({
where: {
path: {
[Op.not]: ourbigbook_paths_converted,
[Op.like]: pathPrefix,
}
}
}),
File.findAll({
where: { path: ourbigbook_paths_converted },
include: [{
model: Render,
where: {
type: Render.Types[nonOurbigbookOptions.renderType],
},
// We still want to get last_parsed from non-rendered files.
required: false,
}],
}),
])
const file_rows_dict = {}
for (const file_row of file_rows) {
file_rows_dict[file_row.path] = file_row
}
nonOurbigbookOptions.file_rows_dict[nonOurbigbookOptions.renderType] = file_rows_dict
}
}
// Do various post conversion checks to verify database integrity:
//
// - duplicate IDs
// - https://docs.ourbigbook.com/x-within-title-restrictions
// - check that all files are included except for the index file
//
// Previously these were done inside ourbigbook.convert. But then we started skipping render by timestamp,
// so if you e.g. move an ID from one file to another, a common operation, then it would still see
// the ID in the previous file depending on conversion order. So we are moving it here instead at the end.
// Having this single query at the end also be slightly more efficient than doing each query separately per file conversion.
async function check_db(nonOurbigbookOptions, { exitOnError=true }={}) {
if (nonOurbigbookOptions.cli.checkDb) {
const t1 = now();
console.log(`check_db`)
const sequelize = nonOurbigbookOptions.sequelize
if (sequelize && (nonOurbigbookOptions.cli.render || nonOurbigbookOptions.cli.checkDb)) {
const error_messages = await ourbigbook_nodejs_webpack_safe.check_db(
sequelize,
nonOurbigbookOptions.ourbigbook_paths_converted,
{
filterFilesThatDontExist: (aRefs) => aRefs.filter(
aRef => !fs.existsSync(resolvePublishedMediaPath(nonOurbigbookOptions.ourbigbook_json_dir, aRef.to, nonOurbigbookOptions))),
options: nonOurbigbookOptions.options,
perf: false,
}
)
if (error_messages.length > 0) {
console.error(error_messages.map(m => 'error: ' + m).join('\n'))
nonOurbigbookOptions.had_error = true
if (exitOnError) cli_error()
return
}
}
console.log(`check_db: ${finished_in_ms(now() - t1)}`)
}
}
function chomp(s) {
return s.replace(/(\r\n|\n)$/, '')
}
/** Report an error with the CLI usage and exit in error. */
function cli_error(message) {
if (message !== undefined) {
console.error(`error: ${message}`)
}
process.exit(1)
}
async function convert_directory_callback(input_path, ourbigbook_options, nonOurbigbookOptions, cb, extraPaths=[]) {
nonOurbigbookOptions.ourbigbook_paths_converted = []
const tasks = []
const sourcePaths = walk_directory_recursively(
input_path,
DEFAULT_IGNORE_BASENAMES_SET,
nonOurbigbookOptions.ignore_paths,
nonOurbigbookOptions.ignore_path_regexps,
nonOurbigbookOptions.dont_ignore_path_regexps,
nonOurbigbookOptions.ourbigbook_json_dir,
)
function* paths() { yield* sourcePaths; yield* extraPaths }
for (const entry of paths()) {
const onePath = typeof entry === 'string' ? entry : entry.path
const virtualPath = typeof entry === 'string' ? undefined : entry.relpath
const relpath = virtualPath ?? path.relative(nonOurbigbookOptions.ourbigbook_json_dir, onePath)
const isPreview = cb === convert_file_preview
const previewPath = path.join(nonOurbigbookOptions.outdir, FILE_PREFIX, relpath + '.' + ourbigbook.HTML_EXT)
if (
nonOurbigbookOptions.parallel &&
!nonOurbigbookOptions.ourbigbook_paths_converted_only &&
(
(isPreview && ourbigbook_options.render && !fs.lstatSync(onePath).isDirectory() &&
!skipFilePreview(onePath, previewPath, relpath, nonOurbigbookOptions)) ||
(cb === convert_path_to_file && isBigbPath(onePath, nonOurbigbookOptions.cli) &&
!do_ignore_convert_path(relpath, nonOurbigbookOptions.ignore_convert_path_regexps,
nonOurbigbookOptions.dont_ignore_convert_path_regexps) &&
!skipFileConversion(onePath, relpath, ourbigbook_options, nonOurbigbookOptions))
)
) {
const row = nonOurbigbookOptions.file_rows_dict[nonOurbigbookOptions.renderType]?.[relpath]
tasks.push({
input_path: onePath,
preview: isPreview,
relpath,
virtualPath,
fileUsageHash: nonOurbigbookOptions.fileUsageHashes?.[FILE_PREFIX + URL_SEP + relpath],
file_row: row ? row.toJSON() : undefined,
render: ourbigbook_options.render,
is_render_after_extract: nonOurbigbookOptions.is_render_after_extract,
})
} else {
await cb(onePath, ourbigbook_options, virtualPath === undefined
? nonOurbigbookOptions : { ...nonOurbigbookOptions, virtualPath })
}
if (nonOurbigbookOptions.had_error) {
break
}
}
if (nonOurbigbookOptions.had_error || !tasks.length) return
// Do not pay worker startup costs for an incremental build of one file.
if (tasks.length === 1 && !nonOurbigbookOptions.parallel.pool) {
await cb(tasks[0].input_path, ourbigbook_options, tasks[0].virtualPath === undefined
? nonOurbigbookOptions : { ...nonOurbigbookOptions, virtualPath: tasks[0].virtualPath })
return
}
const parallel = nonOurbigbookOptions.parallel
if (!parallel.pool) {
const sequelize = nonOurbigbookOptions.sequelize
if (sequelize.getDialect() === 'sqlite') {
// Readers run alongside the single writer without blocking its commit.
await sequelize.query('PRAGMA journal_mode=WAL')
}
const plainOptions = Object.fromEntries(Object.entries(ourbigbook_options).filter(
([key, value]) => !['db_provider', 'katex_macros'].includes(key) && typeof value !== 'function'))
const plainNonOptions = Object.fromEntries(Object.entries(nonOurbigbookOptions).filter(
([key, value]) => !['sequelize', 'options', 'extra_returns', 'parallel', 'file_rows_dict', 'fileUsageHashes', 'localMediaDirectories'].includes(key) &&
typeof value !== 'function'))
parallel.pool = new CliWorkerPool(__filename, nonOurbigbookOptions.cli.jobs, {
options: plainOptions,
nonOptions: plainNonOptions,
})
}
for (const result of await parallel.pool.run(tasks)) {
nonOurbigbookOptions.ourbigbook_paths_converted.push(...result.paths)
nonOurbigbookOptions.sitemap.push(...result.sitemap)
nonOurbigbookOptions.had_error ||= result.had_error
nonOurbigbookOptions.build_errors.push(...result.build_errors)
nonOurbigbookOptions.last_output_path = result.last_output_path
}
}
/**
* @param {String} input_path - path to a directory to convert files in
*/
async function convert_directory(input_path, ourbigbook_options, nonOurbigbookOptions) {
return convert_directory_callback(input_path, ourbigbook_options, nonOurbigbookOptions, convert_path_to_file)
}
/** Extract IDs from all input files into the ID database, without fully converting. */
async function convert_directory_extract_ids(input_path, ourbigbook_options, nonOurbigbookOptions) {
await convert_directory(
input_path,
ourbigbook.cloneAndSet(ourbigbook_options, 'render', false),
nonOurbigbookOptions
)
}
async function convert_directory_extract_ids_and_render(input_dir, ourbigbook_options, nonOurbigbookOptions) {
// Shared by the coordinator's shallow option copies throughout this build.
nonOurbigbookOptions.build_errors = []
if (
nonOurbigbookOptions.cli.jobs > 1 &&
!nonOurbigbookOptions.cli.watch && !nonOurbigbookOptions.cli.formatSource &&
!ourbigbook_options.embed_includes &&
nonOurbigbookOptions.db_options.storage !== ourbigbook_nodejs_webpack_safe.SQLITE_MAGIC_MEMORY_NAME
) {
nonOurbigbookOptions.parallel = {}
}
try {
await convert_directory_stages(input_dir, ourbigbook_options, nonOurbigbookOptions)
} finally {
const pool = nonOurbigbookOptions.parallel?.pool
if (pool) {
await pool.close()
if (nonOurbigbookOptions.sequelize.getDialect() === 'sqlite') {
await nonOurbigbookOptions.sequelize.query('PRAGMA wal_checkpoint(TRUNCATE)')
}
}
delete nonOurbigbookOptions.parallel
if (pool && nonOurbigbookOptions.build_errors.length) {
// Workers inherit stdout/stderr. Wait for their exit before repeating errors,
// so no in-flight progress output can bury the final diagnostics.
console.error('\nBuild failed. Errors:')
for (const error of new Set(nonOurbigbookOptions.build_errors)) console.error(error)
}
}
}
async function convert_directory_stages(input_dir, ourbigbook_options, nonOurbigbookOptions) {
const localMedia = ourbigbook_options.output_format === ourbigbook.OUTPUT_FORMAT_HTML &&
path.resolve(input_dir) === path.resolve(nonOurbigbookOptions.ourbigbook_json_dir)
? collectLocalMedia(nonOurbigbookOptions) : undefined
nonOurbigbookOptions.localMediaDirectories = localMedia?.directories
await reconcile_db_and_filesystem(input_dir, ourbigbook_options, nonOurbigbookOptions)
await convert_directory_extract_ids(input_dir, ourbigbook_options, nonOurbigbookOptions)
if (!nonOurbigbookOptions.had_error) {
// check_db must come before any type of rendering and blow up
// or else it can lead to infinite DB queries notably on duplicate ID scenarios.
if (
!nonOurbigbookOptions.had_error &&
(
ourbigbook_options.render ||
path.relative(nonOurbigbookOptions.ourbigbook_json_dir, input_dir) === ''
)
) {
await check_db(nonOurbigbookOptions, { exitOnError: false })
if (nonOurbigbookOptions.had_error) return
}
if (ourbigbook_options.output_format === ourbigbook.OUTPUT_FORMAT_HTML) {
// File bytes can stay unchanged while their embedding articles change.
// Fetch lightweight usage dependencies once, not once per preview.
const usage = {}
const sequelize = nonOurbigbookOptions.sequelize
if (sequelize) {
const { ARef, File, Id } = sequelize.models
const refs = await ARef.findAll({
attributes: ['to'],
raw: true,
include: [
{ model: File, as: 'aRefDefinedAt', attributes: ['last_parse'], required: true },
{ model: Id, as: 'fromId', attributes: ['idid'], required: true },
],
order: [['to', 'ASC'], [{ model: Id, as: 'fromId' }, 'idid', 'ASC']],
})
for (const ref of refs) {
const target = FILE_PREFIX + URL_SEP + ourbigbook.fileUsagePath(ref.to, nonOurbigbookOptions.localMediaPath)
if (ref['fromId.idid'] === target) continue
;(usage[target] ||= []).push([ref['fromId.idid'], ref['aRefDefinedAt.last_parse']])
}
}
nonOurbigbookOptions.fileUsageHashes = Object.fromEntries(
Object.entries(usage).map(([id, refs]) => [id, hashToHex(JSON.stringify(refs))]))
// Auto-generate a {file} _file page for each file in the project that does not have one already.
// Auto-generate source, and convert it on the fly, a bit like for _dir conversion.
const ourbigbook_paths_converted = nonOurbigbookOptions.ourbigbook_paths_converted
// Not ideal, but we'll do it simple for now. This needs to be restored or a test fails.
nonOurbigbookOptions.ourbigbook_paths_converted = ourbigbook_paths_converted
await convert_directory_callback(
input_dir,
ourbigbook_options,
nonOurbigbookOptions,
convert_file_preview,
localMedia?.files,
)
// Not ideal, but we'll do it simple for now. This needs to be restored or a test fails.
nonOurbigbookOptions.ourbigbook_paths_converted = ourbigbook_paths_converted
}
if (
nonOurbigbookOptions.cli.render &&
!nonOurbigbookOptions.had_error
) {
nonOurbigbookOptions.nonBigbPathsConverted = new Set()
const newNonOurbigbookOptions = ourbigbook.cloneAndSet(nonOurbigbookOptions, 'is_render_after_extract', true)
await convert_directory(
input_dir,
ourbigbook_options,
newNonOurbigbookOptions,
)
nonOurbigbookOptions.had_error = newNonOurbigbookOptions.had_error
if (localMedia && !nonOurbigbookOptions.had_error) {
for (const { path: source, relpath } of localMedia.files) {
const destination = path.join(nonOurbigbookOptions.outdir, RAW_PREFIX, relpath)
if (nonOurbigbookOptions.cli.forceRender || !fs.existsSync(destination) ||
fs.statSync(source).mtimeMs > fs.statSync(destination).mtimeMs) {
fs.mkdirSync(path.dirname(destination), { recursive: true })
fs.copyFileSync(source, destination)
}
}
// Shared directories were rendered in the ordinary source walk above.
for (const relpath of Object.keys(localMedia.directories)) {
if (fs.existsSync(path.join(nonOurbigbookOptions.ourbigbook_json_dir, relpath))) continue
const listingDir = path.join(nonOurbigbookOptions.outdir, DIR_PREFIX, relpath)
if (fs.existsSync(listingDir) && !fs.statSync(listingDir).isDirectory()) {
cli_error(`local media collides with generated output: ${relpath}`)
}
await convert_path_to_file(path.join(localMedia.root, relpath), ourbigbook_options,
{ ...nonOurbigbookOptions, virtualPath: relpath })
}
}
}
}
}
function filePreviewUsageMarker(relpath, nonOptions) {
const hash = nonOptions.fileUsageHash || nonOptions.fileUsageHashes?.[FILE_PREFIX + URL_SEP + relpath] || hashToHex('[]')
return `<!--ourbigbook-file-usage-v3:${hash}-->`
}
function outputHasMarker(outpath, marker) {
if (!fs.existsSync(outpath)) return false
const size = fs.statSync(outpath).size
if (size < marker.length) return false
// Read only the cache marker, even when the preview contains a large file.
const fd = fs.openSync(outpath, 'r')
try {
const tail = Buffer.alloc(marker.length)
fs.readSync(fd, tail, 0, tail.length, size - tail.length)
return tail.toString() === marker
} finally {
fs.closeSync(fd)
}
}
function skipFilePreview(onePath, outpath, relpath, nonOptions) {
return !nonOptions.cli.forceRender && fs.existsSync(outpath) &&
fs.statSync(onePath).mtime <= fs.statSync(outpath).mtime &&
outputHasMarker(outpath, filePreviewUsageMarker(relpath, nonOptions))
}
async function convert_file_preview(onePath, ourbigbook_options, nonOurbigbookOptions) {
const messagePrefix = 'file'
if (
// TODO move dir conversion here, remove this check:
// https://docs.ourbigbook.com/todo/show-directory-listings-on-file-headers
!fs.lstatSync(onePath).isDirectory()
) {
const inputPathRelativeToOurbigbookJson = nonOurbigbookOptions.virtualPath ?? path.relative(nonOurbigbookOptions.ourbigbook_json_dir, onePath);
const outputPathRelativeToOutdir = path.join(
ourbigbook.FILE_PREFIX,
inputPathRelativeToOurbigbookJson + '.' + ourbigbook.HTML_EXT
)
const outpath = path.join(nonOurbigbookOptions.outdir, outputPathRelativeToOutdir)
const msgRet = convert_path_to_file_print_starting(ourbigbook_options, onePath, messagePrefix)
if (
nonOurbigbookOptions.publish &&
ourbigbook_options.ourbigbook_json.generateSitemap &&
ourbigbook_options.render
) {
nonOurbigbookOptions.sitemap.push(outputPathRelativeToOutdir)
}
let skip
if (
skipFilePreview(onePath, outpath, inputPathRelativeToOurbigbookJson, nonOurbigbookOptions)
) {
skip = true
} else {
const sequelize = nonOurbigbookOptions.sequelize
const idid = ourbigbook.FILE_PREFIX + ourbigbook.URL_SEP + inputPathRelativeToOurbigbookJson
if (
!sequelize ||
!(await sequelize.models.Id.findOne({
attributes: ['id'],
// Without splitting, an inline {file} section still needs a preview,
// but an authored standalone file page must never be overwritten.
where: { idid, ...(ourbigbook_options.split_headers ? {} : { toplevel_id: idid }) },
}))
) {
const src = `${ourbigbook.SHORTHAND_HEADER_CHAR} ${ourbigbook.ourbigbookEscapeNotStart(inputPathRelativeToOurbigbookJson)}\n{file}\n`
const inputPath = ourbigbook.FILE_PREFIX + ourbigbook.URL_SEP + inputPathRelativeToOurbigbookJson + '.' + ourbigbook.OURBIGBOOK_EXT
const newOptions = {
...ourbigbook_options,
auto_generated_source: true,
split_headers: false,
hFileShowLarge: true,
}
const newNonOurbigbookOptions = { ...nonOurbigbookOptions }
newNonOurbigbookOptions.input_path = inputPath
const output = await convert_input(src, newOptions, newNonOurbigbookOptions);
if (newNonOurbigbookOptions.had_error) {
throw new Error(`src: ${src}`)
}
if (newOptions.render) {
fs.mkdirSync(path.dirname(outpath), { recursive: true });
fs.writeFileSync(outpath, output + filePreviewUsageMarker(inputPathRelativeToOurbigbookJson, nonOurbigbookOptions))
}
} else {
skip = 'skipped ID already exists'
}
}
convert_path_to_file_print_finish(ourbigbook_options, onePath, outpath, { skip, message_prefix: messagePrefix, t0: msgRet.t0 })
}
}
/** Convert OurBigBook input from a string to output and return the output as a string.
*
* Wraps ourbigbook.convert with CLI usage convenience.
*
* @param {String} input
* @param {Object} options - options to be passed to ourbigbook.convert
* @param {Object} nonOurbigbookOptions - control options for this function,
* not passed to ourbigbook.convert. Also contains some returns:
* - {bool} had_error
* - {Object} extra_returns
* @return {String}
*/
async function convert_input(input, ourbigbook_options, nonOurbigbookOptions={}) {
const new_options = { ...ourbigbook_options }
if ('input_path' in nonOurbigbookOptions) {
new_options.input_path = nonOurbigbookOptions.input_path
}
if ('title' in nonOurbigbookOptions) {
new_options.title = nonOurbigbookOptions.title
}
let inputFormat = nonOurbigbookOptions.input_format || nonOurbigbookOptions.cli.inputFormat
if (inputFormat === undefined && nonOurbigbookOptions.input_path) {
const inputExt = path.parse(nonOurbigbookOptions.input_path).ext
if (inputExt === '.md' || inputExt === '.markdown') {
inputFormat = 'markdown'
}
}
if (inputFormat === undefined) {
inputFormat = 'bigb'
}
if (inputFormat === 'markdown' || inputFormat === 'md') {
input = await ourbigbook.markdownToOurbigbook(input)
} else if (inputFormat !== 'bigb') {
cli_error(`unknown input format: ${inputFormat}`)
}
new_options.extra_returns = {}
// If we don't where the output will go (the case for stdout) or
// the user did not explicitly request full embedding, inline all CSS.
// Otherwise, include and external CSS to make each page lighter.
if (nonOurbigbookOptions.cli.embedResources) {
new_options.template_vars.style = fs.readFileSync(
ourbigbook_nodejs.DIST_CSS_PATH,
ourbigbook_nodejs_webpack_safe.ENCODING
)
new_options.template_vars.post_body = `<script>${fs.readFileSync(
ourbigbook_nodejs.DIST_JS_PATH, ourbigbook_nodejs_webpack_safe.ENCODING)}</script>\n`
} else {
let includes_str = ``;
let scripts_str = ``;
let includes = [];
let scripts = [];
let includes_local = [];
let scripts_local = [];
let template_includes_relative = [];
let template_scripts_relative = [];
if (nonOurbigbookOptions.publish) {
template_includes_relative.push(
path.relative(
nonOurbigbookOptions.outdir,
nonOurbigbookOptions.out_css_path
)
);
template_scripts_relative.push(
path.relative(
nonOurbigbookOptions.outdir,
nonOurbigbookOptions.out_js_path
)
);
} else {
includes_local.push(nonOurbigbookOptions.out_css_path);
scripts_local.push(nonOurbigbookOptions.out_js_path);
}
if (
ourbigbook_options.outfile !== undefined &&
!is_installed_globally
) {
for (const include of includes_local) {
includes.push(path.relative(path.dirname(ourbigbook_options.outfile), include));
}
for (const script of scripts_local) {
scripts.push(path.relative(path.dirname(ourbigbook_options.outfile), script));
}
} else {
includes.push(...includes_local);
scripts.push(...scripts_local);
}
for (const include of includes) {
includes_str += `@import "${cssEscapeDoubleQuotedString(include)}";\n`;
}
for (const script of scripts) {
scripts_str += `<script src="${script}"></script>\n`
}
new_options.template_vars.style = `\n${includes_str}`
new_options.template_vars.post_body = `${scripts_str}`
new_options.template_styles_relative = template_includes_relative;
new_options.template_scripts_relative = template_scripts_relative;
}
// Finally, do the conversion!
const output = await ourbigbook.convert(input, new_options, new_options.extra_returns);
if (nonOurbigbookOptions.post_convert_callback) {
await nonOurbigbookOptions.post_convert_callback(nonOurbigbookOptions.input_path, new_options.extra_returns)
}
if (nonOurbigbookOptions.log.ast) {
console.error('ast:');
console.error(JSON.stringify(new_options.extra_returns.ast, null, 2));
console.error();
}
if (nonOurbigbookOptions.log['ast-simple']) {
console.error('ast-simple:');
console.error(new_options.extra_returns.ast.toString());
console.error();
}
// Remove duplicate messages due to split header rendering. We could not collect
// errors from that case at all maybe, but do we really want to run the risk of
// missing errors?
for (const error_string of ourbigbook_nodejs_webpack_safe.remove_duplicates_sorted_array(
new_options.extra_returns.errors.map(e => e.toString()))) {
console.error(error_string);
nonOurbigbookOptions.build_errors?.push(error_string)
}
for (const warning_string of ourbigbook_nodejs_webpack_safe.remove_duplicates_sorted_array(
new_options.extra_returns.warnings.map(warning => warning.toString()))) {
console.error(warning_string);
}
nonOurbigbookOptions.extra_returns = new_options.extra_returns;
if (new_options.extra_returns.errors.length > 0) {
nonOurbigbookOptions.had_error = true;
}
ourbigbook.perfPrint(new_options.extra_returns.context, 'convert_input_end')
return output;
}
function isBigbPath(input_path, cli) {
const ext = path.extname(input_path)
const markdownExplicit = cli.inputFormat === 'markdown' || cli.inputFormat === 'md'
return ((ext === `.${OURBIGBOOK_EXT}` && !markdownExplicit) || ext === '.md' || ext === '.markdown') &&
!fs.lstatSync(input_path).isDirectory()
}
function skipFileConversion(input_path, relpath, options, nonOptions) {
const row = nonOptions.file_rows_dict[nonOptions.renderType]?.[relpath]
if (!row) return false
if (row.last_parse && new Date(row.last_parse).getTime() <= nonOptions.configMtime) {
if (row.Render) row.Render.outdated = true
return false
}
if (options.render) {
return !!row.Render && !nonOptions.cli.forceRender && !row.Render.outdated && !nonOptions.cli.formatSource
}
const skip = !nonOptions.cli.forceRender && row.last_parse !== null && row.last_parse > fs.statSync(input_path).mtime
if (!skip && row.Render) row.Render.outdated = true
return skip
}
async function withDatabaseWrite(callback) {
if (!isConversionWorker) return callback()
await new Promise(resolve => {
grantDatabaseWrite = resolve
process.send({ acquireWrite: true })
})
try {
return await callback()
} finally {
process.send({ releaseWrite: true })
}
}
/** Convert filetypes that ourbigbook knows how to convert, and just copy those that we don't, e.g.:
*
* * .bigb to .html
* * .scss to .css
*
* @param {string} input_path - path relative to the base_path, e.g. `./ourbigbook subdir` gives:
* base_path: "subdir" and input_path "index.bigb" amongst other files.
*
* The output file name is derived from the input file name with the output extension.
*/
async function convert_path_to_file(input_path, ourbigbook_options, nonOurbigbookOptions={}) {
let msg_ret
let output, first_output_path;
let skip = false
const is_directory = fs.lstatSync(input_path).isDirectory()
let full_path = path.resolve(input_path);
let input_path_parse = path.parse(input_path);
let input_path_relative_to_ourbigbook_json;
if (nonOurbigbookOptions.ourbigbook_json_dir !== undefined) {
input_path_relative_to_ourbigbook_json = nonOurbigbookOptions.virtualPath ?? path.relative(nonOurbigbookOptions.ourbigbook_json_dir, input_path);
}
let new_options
const isbigb = isBigbPath(input_path, nonOurbigbookOptions.cli)
if (is_directory || isbigb) {
nonOurbigbookOptions.ourbigbook_paths_converted.push(input_path_relative_to_ourbigbook_json)
if (nonOurbigbookOptions.ourbigbook_paths_converted_only) {
return
}
new_options = {
...ourbigbook_options
}
}
let showFinish = false
let message_prefix
const ignore_convert_path = do_ignore_convert_path(
input_path_relative_to_ourbigbook_json,
nonOurbigbookOptions.ignore_convert_path_regexps,
nonOurbigbookOptions.dont_ignore_convert_path_regexps
)
if (isbigb) {
if (!ignore_convert_path) {
showFinish = true
msg_ret = convert_path_to_file_print_starting(ourbigbook_options, input_path)
message_prefix = msg_ret.message_prefix
let newNonOurbigbookOptions = { ...nonOurbigbookOptions }
let input = fs.readFileSync(full_path, newNonOurbigbookOptions.encoding);
skip = skipFileConversion(input_path, input_path_relative_to_ourbigbook_json,
ourbigbook_options, nonOurbigbookOptions)
if (!skip) {
newNonOurbigbookOptions.input_path = input_path_relative_to_ourbigbook_json;
// Convert.
const new_options_main = { ...new_options }
let ourbigbook_json = new_options_main.ourbigbook_json
if (ourbigbook_json === undefined) {
ourbigbook_json = {}
} else {
ourbigbook_json = { ...ourbigbook_json }
}
new_options_main.ourbigbook_json = ourbigbook_json
let lint = ourbigbook_json.lint
if (lint === undefined) {
lint = {}
} else {
lint = { ...lint }
}
ourbigbook_json.lint = lint
if (lint.startsWithH1Header === undefined) {
lint.startsWithH1Header = true
}
output = await convert_input(input, new_options_main, newNonOurbigbookOptions);
if (newNonOurbigbookOptions.had_error) {
nonOurbigbookOptions.had_error = true;
}
const extra_returns = newNonOurbigbookOptions.extra_returns
if (
nonOurbigbookOptions.cli.formatSource &&
ourbigbook_options.render
) {
if (!newNonOurbigbookOptions.had_error) {
fs.writeFileSync(full_path, output);
}
first_output_path = full_path
} else {
// Write out the output the output files.
for (const outpath in extra_returns.rendered_outputs) {
const output_path = path.join(nonOurbigbookOptions.outdir, outpath);
if (output_path === full_path) {
cli_error(`output path equals input path: "${outpath}"`);
}
fs.mkdirSync(path.dirname(output_path), { recursive: true });
if (
ourbigbook_options.render &&
nonOurbigbookOptions.publish &&
ourbigbook_options.ourbigbook_json.generateSitemap
) {
nonOurbigbookOptions.sitemap.push(outpath)
}
const renderedOutput = extra_returns.rendered_outputs[outpath]
if (
// We only want split on web.
!nonOurbigbookOptions.web || renderedOutput.split
) {
if (first_output_path === undefined) {
first_output_path = output_path
}
fs.writeFileSync(output_path, renderedOutput.full)
}
}
}
if (
new_options.split_headers &&
ourbigbook_options.output_format === ourbigbook.OUTPUT_FORMAT_HTML
) {
for (const header_ast of extra_returns.context.synonym_headers) {
const new_options_redir = { ...nonOurbigbookOptions.options }
new_options_redir.db_provider = extra_returns.context.db_provider;
await generate_redirect(new_options_redir, header_ast.id, header_ast.synonym, nonOurbigbookOptions.outdir);
}
}
const context = extra_returns.context;
if (nonOurbigbookOptions.log.headers) {
console.error(context.header_tree.toString());
}
// Update the Sqlite database with results from the conversion.
ourbigbook.perfPrint(context, 'convert_path_pre_sqlite')
if ('sequelize' in nonOurbigbookOptions && !nonOurbigbookOptions.options.embed_includes) {
await withDatabaseWrite(() => ourbigbook_nodejs_webpack_safe.update_database_after_convert({
extra_returns,
db_provider: new_options.db_provider,
had_error: nonOurbigbookOptions.had_error,
is_render_after_extract: nonOurbigbookOptions.is_render_after_extract,
nonOurbigbookOptions,
renderType: nonOurbigbookOptions.renderType,
path: input_path_relative_to_ourbigbook_json,
render: ourbigbook_options.render,
sequelize: nonOurbigbookOptions.sequelize,
}))
}
}
} else {
console.log(`ignoreConvert: ${input_path_relative_to_ourbigbook_json}`)
}
}
if (
nonOurbigbookOptions.cli.formatSource
) {
nonOurbigbookOptions.last_output_path = first_output_path
// I should use callbacks instead of doing this. But lazy.
return
}
// _raw and _dir auto-generation. --split-headers on explicit headers with {file}
// still happens outside of this, this is just for auto-generation.
const convertNonBigb = ourbigbook_options.output_format === ourbigbook.OUTPUT_FORMAT_HTML
const editableMarkdown = ourbigbook_options.output_format === ourbigbook.OUTPUT_FORMAT_MARKDOWN
const copyToRaw = ourbigbook_options.output_format !== ourbigbook.OUTPUT_FORMAT_OURBIGBOOK && !(editableMarkdown && isbigb)
let output_path_noext
const forceRender = nonOurbigbookOptions.input_path_is_file || nonOurbigbookOptions.cli.forceRender
if (ourbigbook_options.output_format !== ourbigbook.OUTPUT_FORMAT_OURBIGBOOK) {
output_path_noext = path.join(
editableMarkdown ? '' : (is_directory ? DIR_PREFIX : RAW_PREFIX),
is_directory ? input_path_relative_to_ourbigbook_json
: path.join(path.dirname(input_path_relative_to_ourbigbook_json), path.parse(input_path_relative_to_ourbigbook_json).name),
)
if (is_directory) {
if (
nonOurbigbookOptions.publish &&
ourbigbook_options.ourbigbook_json.generateSitemap &&
ourbigbook_options.render
) {
nonOurbigbookOptions.sitemap.push(output_path_noext + URL_SEP)
}
output_path_noext = path.join(output_path_noext, 'index')
}
if (ourbigbook_options.outfile === undefined) {
output_path_noext = path.join(nonOurbigbookOptions.outdir, output_path_noext);
} else {
output_path_noext = ourbigbook_options.outfile;
}
if (ourbigbook_options.render) {
fs.mkdirSync(path.dirname(output_path_noext), { recursive: true });
if (!isbigb && convertNonBigb && !ignore_convert_path) {
// Convert non-OurBigBook files and directories.
let isSass = false
let knownType = true
if (is_directory) {
first_output_path = path.join(output_path_noext + '.' + ourbigbook.HTML_EXT)
message_prefix = 'dir'
} else {
if (input_path_parse.ext === SASS_EXT) {
isSass = true
first_output_path = output_path_noext + '.css'
message_prefix = 'scss'
} else {
knownType = false
}
}
if (knownType) {
showFinish = true
}
const directoryEntries = is_directory ? mergedDirectoryEntries(
input_path_relative_to_ourbigbook_json, nonOurbigbookOptions) : undefined
const directoryMarker = is_directory
? `<!--ourbigbook-directory:${hashToHex(JSON.stringify(directoryEntries))}-->` : ''
if (
fs.existsSync(first_output_path) &&
fs.statSync(input_path).mtime <= fs.statSync(first_output_path).mtime &&
!forceRender && (!is_directory || outputHasMarker(first_output_path, directoryMarker))
) {
skip = true
} else {
if (knownType) {
msg_ret = convert_path_to_file_print_starting(ourbigbook_options, input_path, message_prefix)
}
if (is_directory) {
// TODO get rid of this, move it entirely to the same code path as {file} generation:
// https://docs.ourbigbook.com/todo/show-directory-listings-on-file-headers
// Generate bigb source code for a directory conversion and render it on the fly.
// TODO move to render https://docs.ourbigbook.com/todo/remove-synthetic-asts
// Not asts, but source code generation here. Even worse! We did it like this to be
// able to more easily reuse the ourbigbook.liquid.html template and style.
//const title = `Directory: ${input_path_relative_to_ourbigbook_json}`
//const arr = [`<!doctype html><html lang=en><head><meta charset=utf-8><title>${title}</title></head><body><h1>${title}</h1><ul>`]
//function push_li(name, isdir) {
// const target = `${name}${isdir ? '/' : ''}`
// arr.push(`<li><a href=${target + (isdir ? 'index.html' : '')}>${target}</a></li>`)
//}
//fs.writeFileSync(output_path, arr.join('') + '</ul></body></html>')
const dirs = []
const files = []
for (const [name, isdir] of directoryEntries) {
;(isdir ? dirs : files).push(name)
}
const dirArr = []
const crumbArr = []
let breadcrumbDir = input_path_relative_to_ourbigbook_json
let upcount = 0
if (breadcrumbDir === '.') {
breadcrumbDir = ''
}
const new_options_dir = {
...new_options,
auto_generated_source: true,
rootRelpathPref: '../',
}
let toplevelId = DIR_PREFIX
if (input_path_relative_to_ourbigbook_json === '') {
new_options_dir.title = ourbigbook.FILE_ROOT_PLACEHOLDER
} else {
new_options_dir.title = input_path_relative_to_ourbigbook_json
toplevelId += ourbigbook.Macro.HEADER_SCOPE_SEPARATOR + input_path_relative_to_ourbigbook_json
}
new_options_dir.toplevelId = toplevelId
const newNonOurbigbookOptions = { ...nonOurbigbookOptions }
// Needed the path to be able to find the relatively placed CSS under _raw.
// notindex.bigb instead of index.bigb because this will be placed at subdir/index.html, unlike the .bigb
// convention that places subdir/index.bigb at subdir.html rather than subdir/index.html so a different
// number of up levels is needed.
newNonOurbigbookOptions.input_path = path.join(DIR_PREFIX, input_path_relative_to_ourbigbook_json, `${INDEX_BASENAME_NOEXT}.${OURBIGBOOK_EXT}`);
const htmlXExtension = newNonOurbigbookOptions.options.htmlXExtension
const indexHtml = htmlXExtension ? 'index.html' : ''
while (true) {
const breadcrumbParse = path.parse(breadcrumbDir)
let bname = breadcrumbParse.name
if (breadcrumbDir === '') {
// TODO https://github.com/ourbigbook/ourbigbook/issues/372
bname = ourbigbook.FILE_ROOT_PLACEHOLDER
}
if (upcount === 0) {
crumbArr.push(ourbigbook.ourbigbookEscapeNotStart(bname))
} else {
crumbArr.push(`\\a[${ourbigbook.ourbigbookEscapeNotStart(path.join(...Array(upcount).fill('..').concat([indexHtml])))}][${ourbigbook.ourbigbookEscapeNotStart(bname)}]{external}`)
}
if (breadcrumbDir === '') {
break
}
breadcrumbDir = breadcrumbParse.dir
upcount++
}
// Root.
dirArr.push(...[...crumbArr].reverse().join(` ${ourbigbook.URL_SEP} `))
dirArr.push(` ${ourbigbook.URL_SEP}`)
if (files.length || dirs.length) {
dirArr.push(`\n\n`)
}
function push_li(name, isdir) {
const target = `${name}`
let targetHref
if (isdir) {
targetHref = target
if (indexHtml) {
targetHref += ourbigbook.URL_SEP + indexHtml
}
} else {
targetHref = path.join(
path.relative(path.join(DIR_PREFIX, input_path_relative_to_ourbigbook_json), '.'),
FILE_PREFIX,
input_path_relative_to_ourbigbook_json,
target
) + (htmlXExtension ? '.' + ourbigbook.HTML_EXT : '')
}
dirArr.push(`* \\a[${ourbigbook.ourbigbookEscapeNotStart(targetHref)}][${ourbigbook.ourbigbookEscapeNotStart(target)}${isdir ? ourbigbook.URL_SEP : ''}]{external}\n`)
}
for (const name of [...dirs].sort()) {
push_li(name, true)
}
for (const name of [...files].sort()) {
push_li(name, false)
}
output = await convert_input(dirArr.join(''), new_options_dir, newNonOurbigbookOptions);
if (newNonOurbigbookOptions.had_error) {
throw new Error()
}
fs.writeFileSync(first_output_path, output + directoryMarker)
} else {
if (isSass) {
fs.writeFileSync(
first_output_path,
require('sass').renderSync({
data: fs.readFileSync(input_path, nonOurbigbookOptions.encoding),
outputStyle: 'compressed',
includePaths: [
path.dirname(ourbigbook_nodejs.PACKAGE_PATH),
],
}).css
);
}
}
}
}
}
}
if (!is_directory && !isbigb && nonOurbigbookOptions.nonBigbPathsConverted !== undefined) {
nonOurbigbookOptions.nonBigbPathsConverted.add(input_path)
}
if (showFinish) {
convert_path_to_file_print_finish(
ourbigbook_options,
input_path,
first_output_path,
{
message_prefix,
skip,
t0: msg_ret ? msg_ret.t0 : undefined
}
)
}
if (ourbigbook_options.output_format !== ourbigbook.OUTPUT_FORMAT_OURBIGBOOK) {
if (!nonOurbigbookOptions.had_error && (isbigb || copyToRaw)) {
// Copy the file into raw.
if (
copyToRaw &&
!nonOurbigbookOptions.ourbigbook_paths_converted_only &&
!is_directory &&
ourbigbook_options.render
) {
const output_path = output_path_noext + input_path_parse.ext;
if (output_path !== path.resolve(input_path)) {
let skip_str
if (
fs.existsSync(output_path) &&
fs.statSync(input_path).mtime <= fs.statSync(output_path).mtime &&
!forceRender
) {
skip_str = ` (${MESSAGE_SKIP_BY_TIMESTAMP})`
} else {
skip_str = ''
}
console.log(`copy: ${path.relative(process.cwd(), input_path)} -> ${path.relative(process.cwd(), output_path)}${skip_str}`)
if (
nonOurbigbookOptions.publish &&
ourbigbook_options.ourbigbook_json.generateSitemap &&
ourbigbook_options.render
) {
nonOurbigbookOptions.sitemap.push(path.relative(nonOurbigbookOptions.outdir, output_path))
}
if (!skip_str) {
fs.copyFileSync(input_path, output_path)
}
}
}
}
}
if (ourbigbook_options.perf) {
console.error(`perf convert_path_to_file_end ${now()}`);
}
nonOurbigbookOptions.last_output_path = first_output_path
return output;
}
function convert_path_to_file_print_starting(ourbigbook_options, input_path, message_prefix) {
if (message_prefix === undefined) {
if (ourbigbook_options.render) {
message_prefix = MESSAGE_PREFIX_RENDER;
} else {
message_prefix = MESSAGE_PREFIX_EXTRACT_IDS;
}
}
const message = `${message_prefix}: ${path.relative(process.cwd(), input_path)}`;
const t0 = now()
console.log(message);
return { message_prefix, t0 };
}
function convert_path_to_file_print_finish(ourbigbook_options, input_path, output_path, opts={}) {
const { message_prefix, skip, t0 } = opts
// Print conversion finished successfully info.
let t1 = now();
let output_path_str
if (
ourbigbook_options.render &&
// Happens if:
// - conversion to .tex
output_path !== undefined
) {
output_path_str = ` -> ${path.relative(process.cwd(), output_path)}`
} else {
output_path_str = ''
}
let skipMsg
if (skip === true) {
skipMsg = MESSAGE_SKIP_BY_TIMESTAMP
} else if (skip) {
skipMsg = skip
}
let doneStr
if (skipMsg) {
doneStr = `(${skipMsg})`
} else {
doneStr = finished_in_ms(t1 - t0)
}
console.log(`${message_prefix}: ${path.relative(process.cwd(), input_path)}${output_path_str} ${doneStr}`);
}
async function create_db(ourbigbook_options, nonOurbigbookOptions) {
perfPrint('create_db_begin', ourbigbook_options)
const sequelize = await ourbigbook_nodejs_webpack_safe.createSequelize(
nonOurbigbookOptions.db_options,
{ force: nonOurbigbookOptions.cli.clearDb },
)
nonOurbigbookOptions.sequelize = sequelize;
ourbigbook_options.db_provider = new ourbigbook_nodejs_webpack_safe.SqlDbProvider(sequelize);
perfPrint('create_db_end', ourbigbook_options)
}
function do_ignore_convert_path(p, ignore_convert_path_regexps, dont_ignore_convert_path_regexps) {
for (const re of ignore_convert_path_regexps) {
if (re.test(p)) {
for (const re2 of dont_ignore_convert_path_regexps) {
if (re2.test(p)) {
return false
}
}
return true
}
}
return false
}
function finished_in_ms(ms) {
return `(finished in ${Math.floor(ms)} ms)`
}
async function generate_redirect(ourbigbook_options, redirect_src_id, redirect_target_id, outdir) {
ourbigbook_options = { ...ourbigbook_options }
ourbigbook_options.input_path = redirect_src_id;
const outpath_basename = redirect_src_id + '.' + ourbigbook.HTML_EXT
const outpath = path.join(outdir, outpath_basename);
ourbigbook_options.outfile = outpath_basename;
const redirect_href = await ourbigbook.convertXHref(redirect_target_id, ourbigbook_options);
if (redirect_href === undefined) {
cli_error(`redirection target ID "${redirect_target_id}" not found`);
}
generate_redirect_base(outpath, redirect_href)
}
function generate_redirect_base(outpath, redirect_href) {
fs.mkdirSync(path.dirname(outpath), {recursive: true})
// https://stackoverflow.com/questions/10178304/what-is-the-best-approach-for-redirection-of-old-pages-in-jekyll-and-github-page/36848440#36848440
fs.writeFileSync(outpath,
`<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Redirecting...</title>
<link rel="canonical" href="${redirect_href}"/>
<meta http-equiv="refresh" content="0;url=${redirect_href}" />
</head>
<body>
<h1>Redirecting...</h1>
<a href="${redirect_href}">Click here if you are not redirected.</a>
<script>location='${redirect_href}'</script>
</body>
</html>
`);
}
/** Return Set of branches in the repository. Hax. */
function git_branches(input_path) {
const str = runCmd('git', ['branch', '-a']).replace(/\n$/, '')
const arr = (str === '') ? [] : str.split('\n');
return new Set(arr.map(s => s.substring(2)));
}
function git_has_commit(input_path) {
try {
runCmd('git', ['-C', input_path, 'log'], { showCmd: false, throwOnError: true })
return true
} catch(err) {
return false
}
}
/**
* Check if path ourbigbook_json_dir is in a git repository and not ignored.
* @return boolean
*/
function git_is_in_repo(ourbigbook_json_dir) {
const extra_returns = {}
runCmd('git', ['-C', ourbigbook_json_dir, 'check-ignore', ourbigbook_json_dir], {
throwOnError: false, showCmd: false, extra_returns })
// Exit statuses:
// - 0: is ignored
// - 1: is not ignored
// - 128: not in git repository
return extra_returns.out.status === 1
}
/**
* @return Array[String] list of all non gitignored files and directories
*/
function git_ls_files(input_path) {
const ret = runCmd(
'git',
['-C', input_path, 'ls-files'],
{
showCmd: false,
throwOnError: true
}
)
ret.replace(/\n$/, '')
if (ret === '') {
return []
} else {
return ret.split('\n')
}
}
/**
* @return {String} full Git SHA of the source.
*/
function gitSha(input_path, srcBranch) {
const args = ['-C', input_path, 'log', '-n1', '--pretty=%H'];
if (srcBranch !== undefined) {
args.push(srcBranch);
}
return chomp(runCmd('git', args, {showCmd: false, throwOnError: true}))
}
// A reusable, disposable checkout. Keep conversion caches, but never source edits.
function checkoutWebSnapshot(source, destination) {
const opts = { throwOnError: true, showCmd: false }
const commit = gitSha(source)
if (!fs.existsSync(destination)) {
fs.mkdirSync(path.dirname(destination), { recursive: true })
runCmd('git', ['clone', '--no-checkout', '--', source, destination], opts)
} else {
const origin = chomp(runCmd('git', ['-C', destination, 'remote', 'get-url', 'origin'], opts))
if (path.resolve(origin) !== path.resolve(source)) cli_error(`unexpected snapshot repository origin: ${destination}`)
runCmd('git', ['-C', destination, 'fetch', 'origin', commit], opts)
}
runCmd('git', ['-C', destination, 'checkout', '--force', '--detach', commit], opts)
runCmd('git', ['-C', destination, 'clean', '-ffd', '-x', '-e', '_out/'], opts)
const updateSubmodules = (original, checkout) => {
const modules = path.join(checkout, '.gitmodules')
if (!fs.existsSync(modules)) return
const entries = runCmd('git', ['config', '--file', modules, '--null', '--get-regexp', '^submodule\\..*\\.path$'], {
...opts, throwOnError: false,
}).split('\0').filter(Boolean)
for (const entry of entries) {
const newline = entry.indexOf('\n')
const relative = entry.slice(newline + 1)
const local = path.join(original, relative)
const nested = path.join(checkout, relative)
const initialized = fs.existsSync(path.join(local, '.git'))
if (initialized) {
// Use local objects even when the pinned commit has not been pushed yet.
// Clone directly: Git's submodule helper rejects paths starting with '-'.
const pinned = chomp(runCmd('git', ['-C', checkout, 'rev-parse', `HEAD:${relative}`], opts))
if (!fs.existsSync(path.join(nested, '.git'))) {
runCmd('git', ['clone', '--no-checkout', '--', local, nested], opts)
} else {
runCmd('git', ['-C', nested, 'fetch', local, pinned], opts)
}
runCmd('git', ['-C', nested, 'checkout', '--force', '--detach', pinned], opts)
} else {
runCmd('git', ['-C', checkout, 'submodule', 'update', '--init', '--force', '--', relative], opts)
}
runCmd('git', ['-C', nested, 'clean', '-ffd', '-x'], opts)
updateSubmodules(local, nested)
}
}
updateSubmodules(source, destination)
return commit
}
function git_toplevel(input_path) {
return chomp(runCmd('git', ['rev-parse', '--show-toplevel'], {
showCmd: false,
throwOnError: true
}))
}
function handleWebApiErr(err) {
if (err.code === 'ECONNREFUSED') {
cli_error('could not connect to server');
} else {
throw err
}
}
// https://stackoverflow.com/questions/37521893/determine-if-a-path-is-subdirectory-of-another-in-node-js
function is_subpath(parent, child) {
const relative = path.relative(parent, child);
return relative && !relative.startsWith('..') && !path.isAbsolute(relative);
}
function perfPrint(name, ourbigbook_options) {
if (ourbigbook_options === undefined || ourbigbook_options.log.perf) {
console.error(`perf ${name} t=${now()}`);
}
}
function relpathCwd(p) {
let ret = path.relative(process.cwd(), p)
if (ret === '')
ret = '.'
return ret
}
/** Render a template file from under template/ */
function renderTemplate(templateRelpath, outdir, env) {
const template = fs.readFileSync(
path.join(ourbigbook_nodejs.PACKAGE_PATH, 'template', templateRelpath),
ourbigbook_nodejs_webpack_safe.ENCODING
);
const out = (new Liquid()).parseAndRenderSync(
template,
env,
{
strictFilters: true,
strictVariables: true,
}
);
fs.writeFileSync(path.join(outdir, templateRelpath), out);
}
function runCmd(cmd, args=[], options={}) {
if (!('dry_run' in options)) {
options.dry_run = false
}
if (!('env_extra' in options)) {
options.env_extra = {}
}
if (!('extra_returns' in options)) {
options.extra_returns = {}
}
if (!('showCmd' in options)) {
options.showCmd = true
}
let { ignoreStdout } = options
if (ignoreStdout === undefined) {
ignoreStdout = false
}
let out
const cmd_str = ([cmd].concat(args)).join(' ')
if (options.showCmd) {
console.log(cmd_str)
}
if (!options.dry_run) {
const spawnOpts = {
cwd: options.cwd,
env: { ...process.env, ...options.env_extra },
}
if (ignoreStdout) {
spawnOpts.stdio = 'ignore'
}
out = child_process.spawnSync(cmd, args, spawnOpts)
}
let ret
if (options.dry_run) {
ret = ''
} else {
if (out.status != 0 && options.throwOnError) {
let msg = `Command failed with status: ${out.status}\ncmd: ${cmd_str}\n`
if (!ignoreStdout) {
if (out.stdout !== null) {
msg += `stdout: \n${out.stdout.toString(ourbigbook_nodejs_webpack_safe.ENCODING)}\n`
}
if (out.stderr !== null) {
msg += `stderr: \n${out.stderr.toString(ourbigbook_nodejs_webpack_safe.ENCODING)}\n`
}
}
throw new Error(msg)
}
if (!ignoreStdout) {
ret = out.stdout.toString(ourbigbook_nodejs_webpack_safe.ENCODING)
}
}
options.extra_returns.out = out
return ret
}
function viewOutput(output_path, cmdOpts) {
const resolved_path = path.resolve(output_path)
if (process.platform === 'darwin') {
runCmd('open', [resolved_path], cmdOpts)
} else if (process.platform === 'win32') {
runCmd('cmd', ['/c', 'start', '', resolved_path], cmdOpts)
} else {
runCmd('xdg-open', [resolved_path], cmdOpts)
}
}
/** Skip path from ourbigbook conversion. */
function ignore_path(
ignore_basenames,
ignore_paths,
ignore_path_regexps,
dont_ignore_path_regexps,
_path
) {
for (const re of dont_ignore_path_regexps) {
if (re.test(_path)) {
return false
}
}
if (
[RAW_PREFIX, DIR_PREFIX, ourbigbook_nodejs.PUBLISH_OBB_PREFIX].some(prefix => _path === prefix || _path.startsWith(prefix + path.sep)) ||
ignore_paths.has(_path) ||
ignore_basenames.has(path.basename(_path))
)
return true
for (const re of ignore_path_regexps) {
if (re.test(_path)) {
return true
}
}
return false
}
/** @alicearmstrong/mathematics.bigb -> mathematics */
function pathNoUsernameNoext(inpath) {
const nousername = inpath.split(ourbigbook.URL_SEP).slice(1).join(ourbigbook.URL_SEP)
return nousername.substr(0, nousername.length - ourbigbook.OURBIGBOOK_EXT.length - 1)
}
/** mathematics -> @alicearmstrong/mathematics.bigb */
function pathUsernameAndExt(username, inpath) {
if (inpath === '') {
inpath = ourbigbook.INDEX_BASENAME_NOEXT
}
return `${ourbigbook.AT_MENTION_CHAR}${username}${ourbigbook.Macro.HEADER_SCOPE_SEPARATOR}${inpath}.${ourbigbook.OURBIGBOOK_EXT}`
}
// Paths that we have determined during ID extraction phase are not modified, so no need for a render stage.
function printStatus({
cleanupDeleted,
i,
inpath,
isUpload,
render,
t0,
t1,
title,
}={}) {
let pref
if (isUpload) {
if (cleanupDeleted) {
pref = 'unlist_upload'
} else {
pref = 'upload'
}
} else {
if (render) {
if (cleanupDeleted) {
pref = 'delete'
} else {
pref = MESSAGE_PREFIX_RENDER
}
} else {
pref = MESSAGE_PREFIX_EXTRACT_IDS
}
}
msg = `web_${pref}: ${i}: ${title ? `${title} ` : ''}${inpath ? `(${inpath})` : ''}`
if (t0 !== undefined) {
msg += ` ${finished_in_ms(t1 - t0)}`
}
console.log(msg)
}
async function updateNestedSet(webApi, username, { foreground=false, watch=true }={}) {
const queued = await queueNestedSetUpdate(webApi, username, { foreground, watch })
if (watch) await waitNestedSetUpdate(webApi, username, queued)
}
async function queueNestedSetUpdate(webApi, username, { foreground=false, watch=true }={}) {
const t1 = now()
console.log(`nested_set`)
const { data, status } = await webApi.articleUpdatedNestedSet(username, { body: { foreground } })
if (status === 202) {
const id = data.job.id
console.log(`nested_set: job ${id} queued${watch ? '; waiting for worker' : ''}`)
return { id, t1 }
}
assertApiStatus(status, data)
return { t1 }
}
async function waitNestedSetUpdate(webApi, username, { id, t1 }) {
if (id !== undefined) {
let previousStatus
let deadline = Date.now() + 21 * 60 * 1000
while (true) {
if (Date.now() > deadline) throw new Error(`Nested-set job ${id} timed out; check worker logs`)
const result = await webApi.articleNestedSetJob(username, id, { timeout: 15000 })
assertApiStatus(result.status, result.data)
const job = result.data.job
if (job.status === 'queued') deadline = Date.now() + 21 * 60 * 1000
if (job.status === 'failed') throw new Error(job.error || `Nested-set job ${id} failed`)
if (job.status === 'completed') break
if (!['queued', 'pending', 'running'].includes(job.status)) throw new Error(`Unknown job status: ${job.status}`)
if (job.status !== previousStatus) {
console.log(`nested_set: job ${id} ${job.status}`)
previousStatus = job.status
}
await new Promise(resolve => setTimeout(resolve, 1000))
}
}
console.log(`nested_set: ${finished_in_ms(now() - t1)}`)
}
class WebBulkBusyError extends Error {}
function checkWebBulkBusy(status, data) {
if (status === 409 && [data && data.errors].flat().includes('Another bulk job is active for this user')) {
throw new WebBulkBusyError('Another bulk job is active for this user')
}
}
async function queueWebArticles(webApi, articles, { phase='render', start=true, batchIndex, batchCount, onQueued, buildId, buildIndex }={}) {
const requestId = require('crypto').randomBytes(16).toString('hex')
const { data, status } = await webApi.articlesBulk(articles, requestId, { timeout: 20000 }, { phase, start, batchIndex, batchCount, buildId, buildIndex })
if (status !== 202) {
checkWebBulkBusy(status, data)
if ([404, 405].includes(status)) console.error('Background rendering requires an updated server. Use --web-individual-upload for the legacy render path.')
assertApiStatus(status, data)
throw new Error(`Expected HTTP 202 when queueing renders, got ${status}`)
}
const id = data.job.id
if (!start) console.log(`web_${phase}_stage: job ${batchIndex + 1}/${batchCount} (${articles.length} articles), first article: ${articles[0].path}`)
if (onQueued) await onQueued()
if (start) await waitWebArticles(webApi, id, phase, { batchIndex, batchCount, firstArticle: articles[0].path })
return id
}
async function waitWebArticles(webApi, id, phase, { allowFailure=false, allowMissing=false, batchIndex, batchCount, firstArticle }={}) {
const startedAt = now()
let previousProgress
let deadline = Date.now() + 21 * 60 * 1000
while (true) {
if (Date.now() > deadline) throw new Error(`Render job ${id} timed out; rerun --web to resume`)
const result = await webApi.articlesBulkJob(id, { timeout: 15000 })
if (allowMissing && result.status === 404) return
assertApiStatus(result.status, result.data)
const job = result.data.job
if (batchIndex === undefined && job.batchIndex != null) batchIndex = job.batchIndex
if (batchCount === undefined && job.batchCount != null) batchCount = job.batchCount
if (['queued', 'waiting'].includes(job.status)) deadline = Date.now() + 21 * 60 * 1000
// An existing job from another CLI invocation has no local batch plan.
const completed = String(job.completed).padStart(String(job.total).length)
const progress = `job ${batchIndex === undefined ? '?' : batchIndex + 1}/${batchCount === undefined ? '?' : batchCount}, article ${completed}/${job.total}, job id: ${id}, status: ${job.status}`
if (progress !== previousProgress) {
deadline = Date.now() + 21 * 60 * 1000
const first = previousProgress === undefined ? `, first article: ${firstArticle || job.firstArticle || 'unknown'}` : ''
const finished = job.status === 'completed' ? ` ${finished_in_ms(now() - startedAt)}` : ''
console.log(`web_${phase}_run: ${progress}${first}${finished}`)
previousProgress = progress
}
if (job.status === 'failed') {
if (!allowFailure) throw new Error(job.error || `Render job ${id} failed`)
console.error(`web_${phase}_run: ${progress}, error: ${job.error || 'unknown error'}`)
return
}
if (job.status === 'completed') break
if (!['waiting', 'queued', 'pending', 'running'].includes(job.status)) throw new Error(`Unknown render job status: ${job.status}`)
await new Promise(resolve => setTimeout(resolve, 1000))
}
}
async function waitWebBuild(webApi, id, { allowFailure=false }={}) {
while (true) {
const { data, status } = await webApi.articlesBuild(id, { timeout: 15000 })
assertApiStatus(status, data)
const build = data.build
if (build.status === 'completed') return
if (build.status === 'failed') {
if (!allowFailure) throw new Error(build.error || 'Build failed')
console.error(`web_upload: build failed: ${build.error || 'unknown error'}`)
return
}
if (build.status !== 'running') throw new Error(`Unexpected build status: ${build.status}`)
if (data.job) await waitWebArticles(webApi, data.job.id, data.job.phase, { allowFailure })
await new Promise(resolve => setTimeout(resolve, 1000))
}
}
// Watch the user's current build, including staging by another CLI. No writes.
async function watchWebBuild(webApi, username) {
let previous
while (true) {
const { data, status } = await webApi.articlesCurrentBuild({ timeout: 15000 })
assertApiStatus(status, data)
const { build, job, tree } = data
let message
if (job) {
await waitWebArticles(webApi, job.id, job.phase, { allowFailure: true, allowMissing: true })
await new Promise(resolve => setTimeout(resolve, 1000))
continue
} else {
message = `web_upload: @${username}: ${tree ? 'tree ' + tree.status : build ? build.status : 'no build'}`
}
if (message !== previous) { console.log(message); previous = message }
if (!job && !tree && (!build || ['completed', 'failed'].includes(build.status))) {
if (build?.status === 'failed') throw new Error(build.error || 'Build failed')
return
}
await new Promise(resolve => setTimeout(resolve, 1000))
}
}
/**
* Walk directory recursively.
*
* https://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
*
* @param {Set} skip_basenames
* @param {Set} ignore_paths
*/
function* walk_directory_recursively(
file_or_dir,
ignore_basenames,
ignore_paths,
ignore_path_regexps,
dont_ignore_path_regexps,
ourbigbook_json_dir
) {
if (!ignore_path(
ignore_basenames,
ignore_paths,
ignore_path_regexps,
dont_ignore_path_regexps,
path.relative(ourbigbook_json_dir, file_or_dir),
)) {
yield file_or_dir;
if (fs.lstatSync(file_or_dir).isDirectory()) {
const dirents = fs.readdirSync(file_or_dir, {withFileTypes: true});
for (const dirent of dirents) {
yield* walk_directory_recursively(
path.join(file_or_dir, dirent.name),
ignore_basenames,
ignore_paths,
ignore_path_regexps,
dont_ignore_path_regexps,
ourbigbook_json_dir,
)
}
}
}
}
async function webCreateOrUpdate({
cleanupDeleted,
fn,
i,
inpath,
isUpload,
render,
title,
webDry,
}) {
printStatus({ i, cleanupDeleted, inpath, isUpload, render, title })
const t0 = now()
let data, status
if (!webDry) {
;({ data, status } = await fn())
assertApiStatus(status, data)
//if (render && i % 100 == 0) {
// runCmd('bin/pg', ['bin/normalize', '-c', '-u', 'cirosantilli', 'nested-set'], {
// cwd: path.join(__dirname, 'web'),
// throwOnError: true,
// })
//}
}
const t1 = now()
printStatus({ i, title, inpath, isUpload, render, cleanupDeleted, t0, t1 })
return { nestedSetNeedsUpdate: data ? data.nestedSetNeedsUpdate : true }
}
// Media has its own physical root but shares the site's logical file namespace.
function collectLocalMedia(nonOptions) {
if (!nonOptions.localMediaPath) return
const root = nonOptions.localMediaSnapshot || path.resolve(nonOptions.ourbigbook_json_dir, nonOptions.localMediaPath)
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
cli_error(`local media directory does not exist: ${root}`)
}
const ignored = git_is_in_repo(root)
? new Set(runCmd('git', ['-C', root, 'ls-files', '--ignored', '--others', '--exclude-standard', '--directory', '-z'],
{ showCmd: false, throwOnError: true }).split('\0').filter(Boolean).map(p => p.replace(/\/$/, '')))
: new Set()
const directories = { '': [] }
const files = []
for (const file of walk_directory_recursively(root,
new Set([...DEFAULT_IGNORE_BASENAMES_SET, '.gitignore', '.gitattributes', '.gitmodules']),
ignored, [], [], root)) {
const relpath = path.relative(root, file)
if (!relpath) continue
const stat = fs.lstatSync(file)
if (stat.isSymbolicLink()) cli_error(`local media symlinks are not supported: ${relpath}`)
const source = path.join(nonOptions.ourbigbook_json_dir, relpath)
if (fs.existsSync(source) && !(stat.isDirectory() && fs.statSync(source).isDirectory())) {
cli_error(`local media collides with source: ${relpath}`)
}
const dirname = path.dirname(relpath)
directories[dirname === '.' ? '' : dirname].push([path.basename(relpath), stat.isDirectory()])
if (stat.isDirectory()) directories[relpath] = []
else if (stat.isFile()) files.push({ path: file, relpath })
}
return { root, directories, files }
}
function mergedDirectoryEntries(relpath, nonOptions) {
const entries = new Map()
const source = path.join(nonOptions.ourbigbook_json_dir, relpath)
if (fs.existsSync(source)) {
for (const entry of fs.readdirSync(source, { withFileTypes: true })) {
if (!ignore_path(DEFAULT_IGNORE_BASENAMES_SET, nonOptions.ignore_paths,
nonOptions.ignore_path_regexps, nonOptions.dont_ignore_path_regexps,
path.join(relpath, entry.name))) {
entries.set(entry.name, entry.isDirectory())
}
}
}
for (const [name, isDirectory] of nonOptions.localMediaDirectories?.[relpath] || []) {
entries.set(name, isDirectory)
}
return [...entries].sort(([a], [b]) => a.localeCompare(b))
}
function pathIsWithin(root, file) {
const relative = path.relative(root, file)
return relative !== '..' && !relative.startsWith('..' + path.sep) && !path.isAbsolute(relative)
}
function resolvePublishedMediaPath(root, file, { localMediaPath, localMediaSnapshot }={}) {
if (localMediaSnapshot) {
const prefix = path.normalize(localMediaPath).replace(/\/$/, '')
if (file === prefix || file.startsWith(prefix + path.sep)) {
return path.join(localMediaSnapshot, path.relative(prefix, file))
}
}
const source = path.join(root, file)
if (localMediaPath && !fs.existsSync(source) && pathIsWithin(root, source)) {
const media = path.join(localMediaSnapshot || path.resolve(root, localMediaPath), file)
if (fs.existsSync(media)) return media
}
return source
}
function fileConversionOptions(ourbigbook_json_dir, mediaOptions) {
const resolveFile = file => resolvePublishedMediaPath(ourbigbook_json_dir, file, mediaOptions)
return {
getAFileTypes: (aRefs) => {
let ret = {}
for (const a of aRefs) {
try {
ret[a] = fs.lstatSync(resolveFile(a)).isFile() ? FILE_TYPE_FILE : FILE_TYPE_DIRECTORY
} catch (error) {
// Missing targets are reported by check_db with their source locations.
// Metadata lookup must not preempt that diagnostic with a Node stack.
if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') throw error
}
}
return ret
},
fs_exists_sync: (my_path) => fs.existsSync(resolveFile(my_path)),
read_include: read_include({
exists: (inpath) => fs.existsSync(path.join(ourbigbook_json_dir, inpath)),
read: (inpath) => fs.readFileSync(path.join(ourbigbook_json_dir, inpath), ourbigbook_nodejs_webpack_safe.ENCODING),
path_sep: ourbigbook.Macro.HEADER_SCOPE_SEPARATOR,
}),
read_file: (readpath, context, opts={}) => {
readpath = resolveFile(readpath)
const mediaRoot = mediaOptions?.localMediaSnapshot || (mediaOptions?.localMediaPath &&
path.resolve(ourbigbook_json_dir, mediaOptions.localMediaPath))
if (
// Let's prevent path transversal a bit by default.
(pathIsWithin(ourbigbook_json_dir, readpath) || (mediaRoot && pathIsWithin(mediaRoot, readpath))) &&
fs.existsSync(readpath)
) {
if (fs.lstatSync(readpath).isFile()) {
return fs.readFileSync(readpath, ourbigbook_nodejs_webpack_safe.ENCODING)
}
} else {
return undefined
}
},
}
}
async function createWebCache(outdir, { sync=true }={}) {
const titleRegex = new RegExp(`${ourbigbook.SHORTHAND_HEADER_CHAR} (.*)`)
const sequelizeWeb = new Sequelize({
dialect: 'sqlite',
storage: path.join(outdir, 'web.sqlite3'),
logging: false,
})
const sequelizeWebArticle = sequelizeWeb.define('Article', {
idid: { type: DataTypes.TEXT, unique: true },
title: { type: DataTypes.TEXT },
body: { type: DataTypes.TEXT },
inpath: { type: DataTypes.TEXT },
parentId: { type: DataTypes.TEXT },
source: { type: DataTypes.TEXT },
definedAt: { type: DataTypes.TEXT },
})
// Just to store the ID of the index.
const sequelizeWebIndexId = sequelizeWeb.define('IndexId', {
idid: { type: DataTypes.TEXT },
// upsert helper.
uniqueHack: { type: DataTypes.INTEGER, unique: true },
})
if (sync) await sequelizeWeb.sync()
const postConvert = async (definedAt, extra_returns) => {
if (extra_returns.errors.length === 0) {
// Use the coordinator's write lock for this second SQLite database too.
// Replace each source file's cached articles in one transaction.
await withDatabaseWrite(() => sequelizeWeb.transaction(async transaction => {
await sequelizeWebArticle.destroy({ where: { definedAt }, transaction })
const rendered_outputs = extra_returns.rendered_outputs
for (let inpath in rendered_outputs) {
const rendered_outputs_entry = rendered_outputs[inpath]
if (rendered_outputs_entry.split) {
// To convert:
//
// linux-kernel-module-cheat-split.bigb
//
// to:
//
// linux-kernel-module-cheat.bigb
//
// on:
//
// = Linux kernel module cheat
// {splitSuffix}
//
// otherwise the ID becomes linux-kernel-module-cheat and \x links fail.
let source = rendered_outputs_entry.full;
const lines = source.split('\n')
let title
if (lines.length) {
const line0 = lines[0]
const titleMatch = line0.match(titleRegex)
if (titleMatch && titleMatch.length >= 2) {
title = titleMatch[1]
}
}
if (title === undefined) {
cli_error(`every bigb must start with a "= Header" for --web upload, failed for: ${inpath}`)
}
const inpathParse = path.parse(inpath)
const pathNoext = path.join(inpathParse.dir, inpathParse.name)
if (rendered_outputs_entry.split_suffix) {
inpath = pathNoext.slice(0, -(rendered_outputs_entry.split_suffix.length + 1)) + `.${ourbigbook.OURBIGBOOK_EXT}`
}
let addId
let addSubdir
let isToplevelIndex = false
const header_ast = rendered_outputs_entry.header_ast
if (ourbigbook.INDEX_FILE_BASENAMES_NOEXT.has(inpathParse.name)) {
if (inpathParse.dir) {
const dirPathParse = path.parse(inpathParse.dir)
const titleId = ourbigbook.titleToId(title)
if (titleId !== dirPathParse.name) {
// This would be ideal, allowing us to store all information about the article in the body itself.
// But it was hard to implement, since now the input path is an important input of conversion.
// So to start with we will just always provide the input path as a separate parameter.
// id= for toplevel was ignored as of writing, which is bad, should be either used or error.
//addId = dirPathParse.name
}
if (dirPathParse.dir) {
// Same as addId
//addSubdir = dirPathParse.dir
}
} else {
// Hack source for subsequent hash calculation to match what we have on server, which
// currently forces "Index" (TODO "index" et al. are likely also possible and would break this hack).
// Ideally we should actually alter the file under _out/web/index.bigb
// but that would be slightly more involved (a new option to convert?) so lazy.
source = source.replace(titleRegex, `${ourbigbook.SHORTHAND_HEADER_CHAR} ${title}`)
await sequelizeWebIndexId.upsert({ idid: header_ast.id, uniqueHack: 0 }, { transaction })
isToplevelIndex = true
}
inpath = `${ourbigbook.INDEX_BASENAME_NOEXT}.${ourbigbook.OURBIGBOOK_EXT}`
} else {
const titleId = ourbigbook.titleToId(title)
if (titleId !== inpathParse.name) {
//addId = inpathParse.name
}
if (inpathParse.dir) {
//addSubdir = inpathParse.dir
}
}
let bodyStart
if (lines[1] === '' && !addId && !addSubdir) {
bodyStart = 2
} else {
bodyStart = 1
}
let body = ''
if (addId) {
// Restore this if we ever remove the separate path magic input.
// Also id of toplevel header is currently ignored as of writing:
//body += `{id=${addId}}\n`
}
if (addSubdir) {
// Restore this if we ever remove the separate path magic input.
//body += `{subdir=${addSubdir}}\n`
}
body += lines.slice(bodyStart).join('\n')
const parent_ast = rendered_outputs_entry.header_ast.get_header_parent_asts(extra_returns.context)[0]
const article = {
body,
inpath,
definedAt,
source,
title,
}
if (parent_ast) {
let parentId
if (
parent_ast.id !== ourbigbook.INDEX_BASENAME_NOEXT &&
// Force every child of the topevel to add it as "@username" and instead of deducing it from title
// as done on CLI. This means that giving the toplevel a custom ID and using that ID will fail to upload...
// there is no solution to that. We should just force the toplevel to have no ID then on CLI for compatibility?
!(
parent_ast.is_first_header_in_input_file &&
ourbigbook.INDEX_FILE_BASENAMES_NOEXT.has(path.parse(parent_ast.source_location.path).name)
)
) {
parentId = `${parent_ast.id}`
} else {
parentId = ''
}
article.parentId = parentId
}
let id_to_article_key
if (isToplevelIndex) {
id_to_article_key = ''
} else {
id_to_article_key = header_ast.id
}
article.idid = id_to_article_key
await sequelizeWebArticle.upsert(article, { transaction })
}
}
}))
}
}
return { sequelize: sequelizeWeb, postConvert }
}
async function runDirectoryWorker() {
// Do not leave workers behind if the coordinator is interrupted or killed.
process.on('disconnect', () => process.exit())
const sequelize = await ourbigbook_nodejs_webpack_safe.createSequelize(
workerData.nonOptions.db_options, {}, { sync: false })
const webCache = workerData.nonOptions.web
? await createWebCache(workerData.nonOptions.outdir, { sync: false }) : undefined
// KaTeX macro tokens contain class instances that cannot cross IPC intact.
const katex_macros = {}
ourbigbook_nodejs_webpack_safe.preload_katex_from_file(ourbigbook_nodejs.DEFAULT_TEX_PATH, katex_macros)
const texPath = path.join(workerData.nonOptions.filesystem_root, OURBIGBOOK_TEX_BASENAME)
if (fs.existsSync(texPath)) ourbigbook_nodejs_webpack_safe.preload_katex_from_file(texPath, katex_macros)
process.on('message', async task => {
try {
if (task.writeGranted) {
grantDatabaseWrite()
return
}
if (task.close) {
if (webCache) await webCache.sequelize.close()
await sequelize.close()
process.disconnect()
return
}
const options = {
...structuredClone(workerData.options),
...fileConversionOptions(workerData.nonOptions.filesystem_root, workerData.nonOptions),
katex_macros,
render: task.render,
// Never carry extraction or another file's cached refs into rendering.
db_provider: new ourbigbook_nodejs_webpack_safe.SqlDbProvider(sequelize),
}
const nonOptions = {
...structuredClone(workerData.nonOptions),
virtualPath: task.virtualPath,
fileUsageHash: task.fileUsageHash,
file_rows_dict: {
[workerData.nonOptions.renderType]: task.file_row ? { [task.relpath]: task.file_row } : {},
},
had_error: false,
build_errors: [],
is_render_after_extract: task.is_render_after_extract,
options: ourbigbook.convertInitOptions(options),
post_convert_callback: webCache?.postConvert,
ourbigbook_paths_converted: [],
sequelize,
sitemap: [],
}
await (task.preview ? convert_file_preview : convert_path_to_file)(task.input_path, options, nonOptions)
process.send({
paths: nonOptions.ourbigbook_paths_converted,
sitemap: nonOptions.sitemap,
had_error: nonOptions.had_error,
build_errors: nonOptions.build_errors,
last_output_path: nonOptions.last_output_path,
})
} catch (error) {
process.send({ error: error.stack || String(error) })
}
})
process.send({ ready: true })
}
if (isConversionWorker) {
process.once('message', data => {
workerData = data
runDirectoryWorker().catch(error => { throw error })
})
} else {
// CLI options.
const cli_parser = commander.program
cli_parser.allowExcessArguments(false);
// Optional arguments.
cli_parser.option('--add-test-instrumentation', 'For testing only', false);
cli_parser.option('--body-only', 'output only the content inside the HTLM body element', false);
cli_parser.option('--check-db-only', `only check the database, don't do anything else: https://docs.ourbigbook.com#check-db`, false);
cli_parser.option('--china', 'https://docs.ourbigbook.com#china', false);
cli_parser.option('--clear-db', 'clear the database before running', false);
cli_parser.option('--dry-run', "don't run most external commands https://github.com/ourbigbook/ourbigbook#dry-run", false);
cli_parser.option('--dry-run-push', "don't run git push commands https://github.com/ourbigbook/ourbigbook#dry-run-push", false);
cli_parser.option('--embed-includes', 'https://docs.ourbigbook.com#embed-include', false);
cli_parser.option('--embed-resources', 'https://docs.ourbigbook.com#embed-resources', false);
// Originally added for testing, this allows the test filesystems to be put under the repository itself,
// otherwise they would pickup our toplevel ourbigbook.json.
cli_parser.option('--escape-literal', 'https://docs.ourbigbook.com#escape-literal', false);
cli_parser.option('--fakeroot <fakeroot>', 'Stop searching for ourbigbook.json at this directory rather than at the filesystem root');
cli_parser.option('--generate <name>', 'https://docs.ourbigbook.com#generate', false);
cli_parser.option('--help-macros', 'print the metadata of all macros to stdout in JSON format. https://docs.ourbigbook.com#help-macros', false);
cli_parser.option('-I, --input-format <input-format>', 'input format: bigb or markdown. Inferred from .md and .markdown extensions by default');
cli_parser.option('-j, --jobs <n>', 'parallel directory conversion workers; defaults to available CPUs: https://docs.ourbigbook.com#jobs', value => {
if (!/^[1-9][0-9]*$/.test(value) || !Number.isSafeInteger(Number(value))) {
throw new commander.InvalidArgumentError('jobs must be a positive integer')
}
return Number(value)
}, os.availableParallelism ? os.availableParallelism() : os.cpus().length || 1);
cli_parser.option('-l, --log <log...>', 'https://docs.ourbigbook.com#log');
cli_parser.option('--no-check-db', 'Skip the database sanity check that is normally done after conversions https://docs.ourbigbook.com#no-check-db');
cli_parser.option('--no-html-x-extension <bool>', 'https://docs.ourbigbook.com#no-html-x-extension', undefined);
cli_parser.option('--no-db', 'ignore the ID database, mostly for testing https://docs.ourbigbook.com#internal-cross-file-references-internals');
cli_parser.option('--no-render', "only extract IDs, don't render: https://docs.ourbigbook.com#no-render");
cli_parser.option('--no-web-render', "same as --no-render, but for --web upload step: https://docs.ourbigbook.com#no-web-render");
cli_parser.option('-F, --force-render', "don't skip render by timestamp: https://docs.ourbigbook.com#no-render-timestamp", false);
cli_parser.option('--outdir <outdir>', 'https://docs.ourbigbook.com#outdir');
cli_parser.option('-o, --outfile <outfile>', 'https://docs.ourbigbook.com#outfile');
cli_parser.option('-O, --output-format <output-format>', 'https://docs.ourbigbook.com#output-format', 'html');
cli_parser.option('-p --publish', 'https://docs.ourbigbook.com#publish', false);
cli_parser.option('--publish-no-convert', 'Attempt to publish without converting. Implies --publish: conversion https://docs.ourbigbook.com#publish-no-convert', false);
cli_parser.option('-P, --publish-commit <commit-message>', 'https://docs.ourbigbook.com#publish-commit');
cli_parser.option('--publish-target <target>', 'https://docs.ourbigbook.com#publish-target', 'github-pages');
cli_parser.option('--format-source', 'https://docs.ourbigbook.com#format-source');
cli_parser.option('-S, --split-headers', 'https://docs.ourbigbook.com#split-headers');
cli_parser.option('--stdout', 'also print output to stdout in addition to saving to a file https://docs.ourbigbook.com#stdout', false);
cli_parser.option('--template <template>', 'https://docs.ourbigbook.com#template');
cli_parser.option('--title-to-id', `read tiles from stdin line by line, output IDs to stdout only, don't do anything else: https://docs.ourbigbook.com#title-to-id`, false);
cli_parser.option('-w, --watch', 'https://docs.ourbigbook.com#watch', false);
cli_parser.option('-W, --web', 'sync to ourbigbook web https://docs.ourbigbook.com#web', false);
cli_parser.option('--web-ask-password', 'Ask the password in case it had some default https://docs.ourbigbook.com#web-ask-password');
cli_parser.option('--web-dry', 'web dry run, skip any --web operations that would interact with the server https://docs.ourbigbook.com#web-dry', false);
cli_parser.option('--web-force-id-extraction', "Force ID extraction on Web: https://docs.ourbigbook.com#web-force-id-extraction");
cli_parser.option('--web-force', 'Replace the current web build without asking for confirmation (implies --web)');
cli_parser.option('--web-cancel-build', 'Cancel the current web build without converting or uploading (implies --web)');
cli_parser.option('--web-watch', 'Only watch the current web build; do not convert or upload (implies --web)');
cli_parser.option('--web-no-watch', 'Exit after submitting the web build, without waiting for completion (implies --web)');
cli_parser.option('--web-force-render', "same as --force-render but for --web upload: https://docs.ourbigbook.com#web-force-render");
cli_parser.option('--web-id <id>', 'Upload only the selected ID. It must belong to a file being converted. https://docs.ourbigbook.com/#web-id');
cli_parser.option('--web-individual-upload', 'Upload articles individually on foreground https://docs.ourbigbook.com/#web-individual-upload');
cli_parser.option('--web-max-renders <n>', 'stop after <n> articles are rendered: https://docs.ourbigbook.com#web-max-renders', cliInt);
cli_parser.option('--web-nested-set', `only update the nested set index, don't do anything else. Implies --web: https://docs.ourbigbook.com#web-nested-set-option`, false);
cli_parser.option('--no-web-nested-set-bulk', `only update the nested set index after all articles have been uploaded: https://docs.ourbigbook.com#web-nested-set-bulk`);
cli_parser.option('--web-password <password>', 'Set password from CLI. Really bad idea for non-test users due e.g. to Bash history: https://docs.ourbigbook.com#web-user');
cli_parser.option('--web-start-id <id>', 'Start the Web article upload passes from this ID, inclusive: https://docs.ourbigbook.com#web-start-id');
cli_parser.option('--web-test', 'Convenient --web-* defaults local development: https://docs.ourbigbook.com#web-test', false);
cli_parser.option('--web-url <url>', 'Set a custom sync URL for --web: https://docs.ourbigbook.com#web-url');
cli_parser.option('--web-user <username>', 'Set username from CLI: https://docs.ourbigbook.com#web-user');
cli_parser.option('--unsafe-ace', 'https://docs.ourbigbook.com#unsafe-ace');
cli_parser.option('--unsafe-xss', 'https://docs.ourbigbook.com#unsafe-xss');
cli_parser.option('-V, --view-output', 'Open the main output file in a browser after a single-file conversion: https://docs.ourbigbook.com#view-output', false);
// Positional arguments.
cli_parser.argument('[input_path...]', 'file or directory to convert http://docs.ourbigbook.com#ourbigbook-executable. If the first path is a file, all others must also be files (and not directories) as an optimization limitation. And they must lie in the same OurBigBook project.');
// Parse CLI.
cli_parser.parse(process.argv);
let [inputPaths] = cli_parser.processedArgs
const cli = cli_parser.opts()
// main action.
;(async () => {
if (cli.helpMacros) {
console.log(JSON.stringify(ourbigbook.macroList(), null, 2));
} else if (cli.china) {
console.log(china_dictatorship.get_data());
} else {
let input;
let output;
let publish = cli.publish || cli.publishCommit !== undefined || cli.publishNoConvert
let htmlXExtension;
let publishTargetIsWebsite
let input_dir;
const web = cli.web || cli.webTest || cli.webNestedSet || cli.webWatch || cli.webNoWatch || cli.webForce || cli.webCancelBuild
if (cli.webNoWatch) {
for (const [option, enabled] of [['--web-watch', cli.webWatch], ['--web-individual-upload', cli.webIndividualUpload]]) {
if (enabled) cli_error(`--web-no-watch cannot be combined with ${option}`)
}
}
if (cli.webStartId !== undefined) {
if (!web) {
cli_error('--web-start-id requires --web or --web-test')
}
if (cli.webId !== undefined) {
cli_error('--web-start-id cannot be combined with --web-id')
}
}
if (inputPaths.length === 0) {
if (web || publish || cli.watch || cli.generate || cli.checkDbOnly || cli.webNestedSet) {
inputPaths = ['.'];
}
} else {
if (cli.generate) {
cli_error('cannot give an input path with --generate');
}
}
// Determine the ourbigbook.json file by walking up the directory tree.
let input_path_is_file;
let inputPath
if (inputPaths.length === 0) {
// Input from stdin.
input_dir = undefined;
input_path_is_file = false;
} else {
for (const inputPath of inputPaths) {
if (!fs.existsSync(inputPath)) {
cli_error('input path does not exist: ' + inputPath);
}
if (input_path_is_file && !fs.lstatSync(inputPath).isFile()) {
cli_error(`the first input path is a file, but one of the other ones isn't: "{inputPath}"`);
}
}
inputPathCwd = relpathCwd(inputPaths[0])
inputPath = inputPaths[0]
input_path_is_file = fs.lstatSync(inputPath).isFile();
if (input_path_is_file) {
input_dir = path.dirname(inputPath);
} else {
input_dir = inputPath;
}
}
// Initialize ourbigbook.json and directories determined from it if present.
let ourbigbook_json_dir
let ourbigbook_json = {}
if (inputPaths.length === 0) {
ourbigbook_json_dir = '.'
} else {
let curdir = path.resolve(inputPath);
if (input_path_is_file) {
curdir = path.dirname(curdir)
}
ourbigbook_json_dir = ourbigbook_nodejs_webpack_safe.findOurbigbookJsonDir(
curdir,
{ fakeroot: cli.fakeroot === undefined ? undefined : path.resolve(cli.fakeroot) },
)
if (ourbigbook_json_dir === undefined) {
// No ourbigbook.json found.
const cwd = process.cwd();
if (is_subpath(cwd, inputPath)) {
ourbigbook_json_dir = cwd
} else {
if (input_path_is_file) {
ourbigbook_json_dir = path.dirname(inputPath)
} else {
ourbigbook_json_dir = inputPath
}
}
} else if (!web || !git_is_in_repo(ourbigbook_json_dir)) {
Object.assign(ourbigbook_json, JSON.parse(fs.readFileSync(
path.join(ourbigbook_json_dir, ourbigbook.OURBIGBOOK_JSON_BASENAME), ourbigbook_nodejs_webpack_safe.ENCODING)))
}
}
let webSourceRoot, webMediaSnapshot
if (web && git_is_in_repo(ourbigbook_json_dir)) {
const opts = { throwOnError: true, showCmd: false }
const source = chomp(runCmd('git', ['-C', ourbigbook_json_dir, 'rev-parse', '--show-toplevel'], opts))
webSourceRoot = path.resolve(ourbigbook_json_dir)
const snapshot = path.join(webSourceRoot, ourbigbook_nodejs_webpack_safe.TMP_DIRNAME, 'publish')
const project = path.join(snapshot, path.relative(source, webSourceRoot))
const absoluteInputs = inputPaths.map(p => path.resolve(p))
for (const p of absoluteInputs) {
if (path.relative(source, p).startsWith('..' + path.sep)) cli_error(`web input is outside the Git repository: ${p}`)
}
if (cli.outdir !== undefined) cli.outdir = path.resolve(cli.outdir)
if (cli.template !== undefined) cli.template = path.join(snapshot, path.relative(source, path.resolve(cli.template)))
const commit = checkoutWebSnapshot(source, snapshot)
console.log(`Web upload uses committed source: ${commit} (checkout: ${project})`)
process.chdir(project)
inputPaths = absoluteInputs.map(p => path.relative(project, path.join(snapshot, path.relative(source, p))) || '.')
inputPath = inputPaths[0]
inputPathCwd = relpathCwd(inputPath)
input_dir = input_path_is_file ? path.dirname(inputPath) : inputPath
ourbigbook_json_dir = project
const config = path.join(project, ourbigbook.OURBIGBOOK_JSON_BASENAME)
ourbigbook_json = fs.existsSync(config) ? JSON.parse(fs.readFileSync(config, 'utf8')) : {}
const mediaPath = ourbigbook_json['media-providers']?.local?.path
if (mediaPath) {
const mediaSource = path.resolve(webSourceRoot, mediaPath)
// Submodules inside the source checkout already use the superproject's pinned commit.
if (!is_subpath(source, mediaSource)) {
if (!git_is_in_repo(mediaSource)) cli_error(`local media provider must be a Git repository for committed web uploads: ${mediaSource}`)
webMediaSnapshot = fs.mkdtempSync(path.join(os.tmpdir(), 'ourbigbook-web-media-'))
process.on('exit', () => fs.rmSync(webMediaSnapshot, { recursive: true, force: true }))
runCmd('git', ['clone', '--shared', '--no-checkout', '--', mediaSource, webMediaSnapshot], opts)
runCmd('git', ['-C', webMediaSnapshot, 'checkout', '--detach', gitSha(mediaSource)], opts)
}
}
}
const publish_target = cli.publishTarget
const publish_target_options = publish && ourbigbook_json.target
? ourbigbook_json.target[publish_target]
: undefined
if (
publish_target_options
) {
ourbigbook_json = lodash.merge(
{},
ourbigbook_json,
publish_target_options
)
}
if (web) {
let jsonH = ourbigbook_json.h
if (jsonH === undefined) {
jsonH = {}
ourbigbook_json.h = jsonH
}
jsonH.splitDefault = true
}
if (
fs.existsSync(DEFAULT_TEMPLATE_BASENAME) &&
!('template' in ourbigbook_json)
) {
ourbigbook_json.template = DEFAULT_TEMPLATE_BASENAME
}
let markdownMaxBytes, split_headers, publish_uses_git;
// Content will become publicly visible after publishing. For example:
// - publish to github pages: yes
// - publish a local file to then ZIP: no
let publishIsPublic
const publish_create_files = {}
if (publish) {
switch (publish_target) {
case 'github-pages':
htmlXExtension = false;
split_headers = true;
publish_uses_git = true;
publishTargetIsWebsite = true
publishIsPublic = true
// Disable Jekyll processing, including its exclusion of underscore paths.
publish_create_files['.nojekyll'] = ''
const cname_path = path.join(ourbigbook_json_dir, 'CNAME')
if (fs.existsSync(cname_path)) {
publish_create_files['CNAME'] = fs.readFileSync(cname_path, ourbigbook_nodejs_webpack_safe.ENCODING)
}
break;
case 'github-md':
if (!publish_target_options || publish_target_options.generateSitemap === undefined) {
ourbigbook_json.generateSitemap = false
}
cli.outputFormat = ourbigbook.OUTPUT_FORMAT_GITHUB_MARKDOWN
htmlXExtension = true
split_headers = true
publish_uses_git = true
publishTargetIsWebsite = false
publishIsPublic = true
markdownMaxBytes = ourbigbook.GITHUB_MARKDOWN_MAX_BYTES
if (ourbigbook_json.githubMarkdownMaxBytes !== undefined) {
if (
!Number.isInteger(ourbigbook_json.githubMarkdownMaxBytes) ||
ourbigbook_json.githubMarkdownMaxBytes <= 0
) {
cli_error('githubMarkdownMaxBytes must be a positive integer')
}
// A project may choose a more conservative limit, but cannot disable
// the GitHub rendering safeguard by raising it.
markdownMaxBytes = Math.min(
markdownMaxBytes,
ourbigbook_json.githubMarkdownMaxBytes,
)
}
break
case 'local':
htmlXExtension = true;
publish_uses_git = false;
publishTargetIsWebsite = false
publishIsPublic = false
break;
default:
cli_error(`unknown publish target: ${publish_target}`)
}
}
if (split_headers === undefined) {
if (cli.splitHeaders === true) {
split_headers = cli.splitHeaders
} else {
split_headers = ourbigbook_json.splitHeaders
}
}
if (htmlXExtension === undefined) {
if (cli.htmlXExtension === false) {
htmlXExtension = cli.htmlXExtension
} else {
htmlXExtension = ourbigbook_json.htmlXExtension
}
}
// Options that will be passed directly to ourbigbook.convert().
if (!(cli.outputFormat in ourbigbook.OUTPUT_FORMATS)) {
cli_error(`unknown output format: ${cli.outputFormat}`)
}
const output_format = (cli.formatSource || web) ? ourbigbook.OUTPUT_FORMAT_OURBIGBOOK : cli.outputFormat
const ourbigbook_options = {
...fileConversionOptions(ourbigbook_json_dir),
add_test_instrumentation: cli.addTestInstrumentation,
body_only: cli.bodyOnly,
ourbigbook_json,
embed_includes: cli.embedIncludes,
htmlXExtension,
markdownIndexBasename: publish_target === 'github-md' ? 'README' : undefined,
markdownMaxBytes,
output_format,
outfile: cli.outfile,
path_sep: path.sep,
publish,
render: cli.render,
showSplitOnToc: ourbigbook_json.showSplitOnToc,
split_headers: split_headers,
template_vars: {
publishTargetIsWebsite: false,
},
unsafeXss: cli.unsafeXss,
webLocalConvert: web,
}
// Resolved options.
const options = ourbigbook.convertInitOptions(ourbigbook_options)
const localMediaPath = options.ourbigbook_json['media-providers'].local.path
const localMediaDir = webMediaSnapshot || (localMediaPath ? path.resolve(ourbigbook_json_dir, localMediaPath) : undefined)
ourbigbook_options.log = {};
const nonOurbigbookOptions_log = {};
if (cli.log !== undefined) {
for (const log of cli.log) {
if (ourbigbook.LOG_OPTIONS.has(log)) {
ourbigbook_options.log[log] = true;
} else if (LOG_OPTIONS.has(log)) {
nonOurbigbookOptions_log[log] = true;
} else {
cli_error('unknown --log option: ' + log);
}
}
}
if (inputPath !== undefined) {
let template_path;
if (cli.template !== undefined) {
template_path = cli.template;
} else if ('template' in ourbigbook_json && ourbigbook_json.template !== null) {
template_path = path.join(ourbigbook_json_dir, ourbigbook_json.template);
}
if (template_path === undefined) {
ourbigbook_options.template = undefined;
} else {
ourbigbook_options.template = fs.readFileSync(template_path).toString();
}
}
if (inputPath !== undefined) {
try {
ourbigbook_options.template_vars.git_sha = gitSha(input_dir);
} catch(error) {
// Not in a git repo.
}
}
let outdir;
if (cli.outdir === undefined) {
if (cli.generate) {
outdir = '.'
} else {
outdir = ourbigbook_json_dir;
}
} else {
outdir = cli.outdir;
}
if (cli.generate) {
let generate = cli.generate
if (generate === 'subdir') {
outdir = path.join(outdir, 'docs')
}
fs.mkdirSync(outdir, {recursive: true});
// Generate package.json.
const package_json = JSON.parse(fs.readFileSync(
ourbigbook_nodejs.PACKAGE_PACKAGE_JSON_PATH).toString());
const package_json_str = `{
"dependencies": {
"ourbigbook": "${package_json.version}"
}
}
`;
fs.writeFileSync(path.join(outdir, 'package.json'), package_json_str);
// Generate .gitignore. Reuse our gitignore up to the first blank line.
let gitignore_new = '';
const gitignore = fs.readFileSync(
ourbigbook_nodejs.GITIGNORE_PATH,
ourbigbook_nodejs_webpack_safe.ENCODING
);
for (const line of gitignore.split('\n')) {
if (line === '') {
break;
}
gitignore_new += line + '\n';
}
fs.writeFileSync(path.join(outdir, '.gitignore'), gitignore_new);
let title = 'Ourbigbook Template';
let multifile
if (generate === 'default') {
renderTemplate(`not-index.${ourbigbook.OURBIGBOOK_EXT}`, outdir, {});
multifile = true
} else {
title += ' ' + generate
multifile = false
}
renderTemplate('README.md', outdir, {})
renderTemplate(`${ourbigbook.INDEX_BASENAME_NOEXT}.${ourbigbook.OURBIGBOOK_EXT}`, outdir, {
generate,
multifile,
title,
version: package_json.version,
});
if (multifile) {
fs.copyFileSync(path.join(ourbigbook_nodejs.PACKAGE_PATH, DEFAULT_TEMPLATE_BASENAME),
path.join(outdir, DEFAULT_TEMPLATE_BASENAME));
fs.copyFileSync(path.join(ourbigbook_nodejs.PACKAGE_PATH, 'main.scss'),
path.join(outdir, 'main.scss'));
fs.copyFileSync(ourbigbook_nodejs.LOGO_PATH, path.join(outdir, ourbigbook_nodejs.LOGO_BASENAME));
}
fs.writeFileSync(
path.join(
outdir,
ourbigbook.OURBIGBOOK_JSON_BASENAME
),
JSON.stringify({
ignore: [
'CONTRIBUTING\\.md',
'README\\.md',
],
}, null, 2) + '\n'
)
process.exit(0)
}
let tmpdir, renderType
const outputOutOfTree = ourbigbook_json.outputOutOfTree !== false || web
if (
// Possible on intput from stdin.
outdir !== undefined
) {
tmpdir = path.join(outdir, ourbigbook_nodejs_webpack_safe.TMP_DIRNAME);
if (
cli.outdir === undefined &&
outputOutOfTree
) {
let subdir
if (web) {
subdir = ourbigbook.RENDER_TYPE_WEB
} else {
subdir = output_format
}
outdir = path.join(tmpdir, subdir)
}
}
if (web) {
renderType = ourbigbook.RENDER_TYPE_WEB
} else {
renderType = output_format
}
// Options that are not directly passed to ourbigbook.convert
// but rather used only by this ourbigbook executable.
const nonOurbigbookOptions = {
ourbigbook_json_dir,
filesystem_root: ourbigbook_json_dir,
localMediaPath,
ourbigbook_paths_converted: [],
ourbigbook_paths_converted_only: false,
cli,
db_options: {},
dont_ignore_path_regexps: options.ourbigbook_json.dontIgnore.map(p => RegExp(`^${p}$`)),
dont_ignore_convert_path_regexps: options.ourbigbook_json.dontIgnoreConvert.map(p => RegExp(
`^${p}($|${lodash.escapeRegExp(path.sep)})`)),
file_rows_dict: {},
encoding: ourbigbook_nodejs_webpack_safe.ENCODING,
external_css_and_js: false,
filterFilesThatDontExist: (aRefs) => aRefs.filter(
aRef => !fs.existsSync(path.join(ourbigbook_json_dir, aRef.to))),
had_error: false,
is_render_after_extract: false,
ignore_path_regexps: options.ourbigbook_json.ignore.map(p => RegExp(
`^${p}($|${lodash.escapeRegExp(path.sep)})`)),
ignore_convert_path_regexps: options.ourbigbook_json.ignoreConvert.map(p => RegExp(
`^${p}($|${lodash.escapeRegExp(path.sep)})`)),
ignore_paths: new Set(),
input_path_is_file,
log: nonOurbigbookOptions_log,
// undefined: don't collect. Set(): collect.
nonBigbPathsConverted: undefined,
// Resolved options.
options,
out_css_path: ourbigbook_nodejs.DIST_CSS_PATH,
out_js_path: ourbigbook_nodejs.DIST_JS_PATH,
outdir,
post_convert_callback: undefined,
publish,
renderType,
sitemap: [],
web,
};
if (publish) {
ourbigbook_options.logoPath = ourbigbook_nodejs.LOGO_ROOT_RELPATH
} else {
ourbigbook_options.logoPath = ourbigbook_nodejs.LOGO_PATH
}
// CLI options
if (webMediaSnapshot) {
Object.assign(nonOurbigbookOptions, { localMediaPath, localMediaSnapshot: webMediaSnapshot })
}
Object.assign(ourbigbook_options, fileConversionOptions(ourbigbook_json_dir, nonOurbigbookOptions))
const cmdOpts = {
dry_run: cli.dryRun,
env_extra: {},
throwOnError: true,
}
const cmdOptsNoDry = { ...cmdOpts }
cmdOptsNoDry.dry_run = false
// Commands that retrieve information and don't change state.
const cmdOptsInfo = { ...cmdOptsNoDry }
cmdOptsInfo.showCmd = false
const cmdOptsInfoNothrow = { ...cmdOptsInfo }
cmdOptsInfoNothrow.throwOnError = false
// We've started using this variatnt for commands that might blow spawnSync stdout buffer size.
// This is not ideal as it prevents obtaining the error messages from stdout/stderr for debug purposes.
// A better solution might instead be to have an async readline variant:
// https://stackoverflow.com/questions/63796633/spawnsync-bin-sh-enobufs/77420941#77420941
const cmdOptsNoStdout = { ...cmdOpts }
cmdOptsNoStdout.ignoreStdout = true
const isInGitRepo = git_is_in_repo(ourbigbook_json_dir)
const configPath = path.join(ourbigbook_json_dir, ourbigbook.OURBIGBOOK_JSON_BASENAME)
nonOurbigbookOptions.configMtime = fs.existsSync(configPath) ? fs.statSync(configPath).mtimeMs : 0
if (localMediaDir) {
const mediaRelpath = path.relative(ourbigbook_json_dir, localMediaDir)
nonOurbigbookOptions.ignore_paths.add(mediaRelpath)
if (mediaRelpath && pathIsWithin(ourbigbook_json_dir, localMediaDir)) {
// This also applies to explicitly selected paths, not just directory walks.
nonOurbigbookOptions.ignore_convert_path_regexps.push(
new RegExp(`^${lodash.escapeRegExp(mediaRelpath)}($|${lodash.escapeRegExp(path.sep)})`))
}
}
if (isInGitRepo && inputPath !== undefined) {
const inputRelpath = path.relative(ourbigbook_json_dir, input_dir)
nonOurbigbookOptions.ignore_paths = new Set([
...nonOurbigbookOptions.ignore_paths,
...runCmd(
'git', ['-C', input_dir, 'ls-files', '--ignored', '--others', '--exclude-standard', '--directory'], cmdOptsInfo
).split('\n').slice(0, -1).map(s => s.replace(/\/$/, '')).map(s => path.join(inputRelpath, s))
])
}
// A custom output directory inside the source tree must not become input on
// later passes, notably once it contains the generated -/ namespace.
const outputRelpath = path.relative(ourbigbook_json_dir, outdir)
if (outputRelpath && outputRelpath !== '..' && !outputRelpath.startsWith('..' + path.sep) && !path.isAbsolute(outputRelpath)) {
nonOurbigbookOptions.ignore_paths.add(outputRelpath)
} else if (outputRelpath === '') {
nonOurbigbookOptions.ignore_paths.add('-')
}
ourbigbook_options.outdir = path.relative(outdir, ourbigbook_json_dir)
if (!nonOurbigbookOptions_log.db) {
// They do not like true, has to be false or function.
// And setting undefined is also considered true.
nonOurbigbookOptions.db_options.logging = false;
}
let input_git_toplevel;
let subdir_relpath;
let publish_tmpdir;
// Load built-in math defines.
const katex_macros = {}
ourbigbook_nodejs_webpack_safe.preload_katex_from_file(ourbigbook_nodejs.DEFAULT_TEX_PATH, katex_macros)
ourbigbook_options.katex_macros = katex_macros
if (cli.titleToId) {
const readline = require('readline');
for await (const line of readline.createInterface({ input: process.stdin })) {
console.log(ourbigbook.titleToId(line))
}
process.exit(0)
}
if (inputPath === undefined) {
if (cli.viewOutput) {
cli_error('--view-output can only be used on single-file conversions');
}
// Input from stdin.
title = 'stdin';
input = await readStdin();
if (cli.escapeLiteral) {
output = ourbigbook.ourbigbookEscape(input)
} else {
output = await convert_input(input, ourbigbook_options, nonOurbigbookOptions);
}
} else {
if (!fs.existsSync(inputPath)) {
cli_error(`input_path does not exist: "${inputPath}"`);
}
if (cli.viewOutput) {
if (!input_path_is_file) {
cli_error('--view-output can only be used on single-file conversions');
}
if (inputPaths.length !== 1) {
cli_error('--view-output requires exactly one input file');
}
if (!cli.render) {
cli_error('--view-output requires render output; it is incompatible with --no-render');
}
}
let publishDir
let publishDirCwd
if (!input_path_is_file) {
if (cli.outfile !== undefined) {
cli_error(`--outfile given but multiple output files must be generated, maybe you want --outdir?`);
}
if (publish) {
input_git_toplevel = git_toplevel(inputPath);
subdir_relpath = path.relative(input_git_toplevel, inputPath);
publishDir = path.join(tmpdir, 'publish');
publishDirCwd = relpathCwd(publishDir)
publish_git_dir = path.join(publishDir, '.git');
if (fs.existsSync(publish_git_dir)) {
// This cleanup has to be done before the database initialization.
runCmd('git', ['-C', publishDirCwd, 'clean', '-x', '-d', '-f'], cmdOpts);
}
publish_tmpdir = path.join(publishDir, subdir_relpath, ourbigbook_nodejs_webpack_safe.TMP_DIRNAME);
}
}
if (publish_tmpdir === undefined) {
publish_tmpdir = tmpdir;
}
// ourbigbook.tex custom math defines.
let tex_path = path.join(ourbigbook_json_dir, OURBIGBOOK_TEX_BASENAME);
if (fs.existsSync(tex_path)) {
ourbigbook_nodejs_webpack_safe.preload_katex_from_file(tex_path, katex_macros)
}
// Setup the ID database.
if (cli.db) {
nonOurbigbookOptions.db_options.storage = path.join(publish_tmpdir, ourbigbook_nodejs_front.SQLITE_DB_BASENAME)
} else {
nonOurbigbookOptions.db_options.storage = ourbigbook_nodejs_webpack_safe.SQLITE_MAGIC_MEMORY_NAME
}
if (cli.checkDbOnly) {
await create_db(ourbigbook_options, nonOurbigbookOptions);
await check_db(nonOurbigbookOptions)
} else if (web) {
let token
let webUrl
if (cli.webUrl) {
webUrl = cli.webUrl
} else if (cli.webTest) {
webUrl = 'http://localhost:3000'
} else {
let host
if (options.ourbigbook_json.web && options.ourbigbook_json.web.host) {
host = options.ourbigbook_json.web.host
} else {
host = ourbigbook.OURBIGBOOK_JSON_DEFAULT.web.host
}
webUrl = `https://${host}`
}
console.log(`Publishing to: ${webUrl}`)
const url = new URL(webUrl)
const host = url.host
await create_db(ourbigbook_options, nonOurbigbookOptions);
// Get username, password and attempt login before anything else.
let username, webApi
if (cli.webUser) {
username = cli.webUser
} else {
if (cli.webTest) {
username = 'barack-obama'
}
}
const cliWhere = { host }
if (username) {
cliWhere.username = username
} else {
cliWhere.defaultUsernameForHost = true
}
const host_row = await nonOurbigbookOptions.sequelize.models.Cli.findOne({ where: cliWhere })
if (username === undefined) {
if (host_row === null) {
;[err, username] = await read({ prompt: 'Username: ' })
} else {
username = host_row.username
console.log(`Using previous username: ${username}\n`);
}
}
webApi = new WebApi({
getToken: () => token,
https: url.protocol === 'https:',
port: url.port,
hostname: url.hostname,
retries: WEB_MAX_RETRIES,
retryDelayMs: WEB_RETRY_DELAY_MS,
validateStatus: () => true,
})
let tokenOk = true
if (host_row) {
token = host_row.token
let data, status
// Can fail with "jwt expired" if expered if you wait for a long time
// after the previous login. So we test the token first thing.
;({ data, status } = await webApi.min())
tokenOk = data.loggedIn
}
if (!host_row || !tokenOk) {
let err, password
// Password
if (cli.webPassword) {
password = cli.webPassword
} else {
if (cli.webTest && !cli.webAskPassword) {
password = 'asdf'
} else {
;[err, password] = await read({ prompt: 'Password: ', silent: true })
}
}
if (!cli.webDry) {
let data, status
try {
;({ data, status } = await webApi.userLogin({ username, password }))
} catch(err) {
handleWebApiErr(err)
}
if (status === 422) {
cli_error('invalid username or password');
} else if (status !== 200) {
cli_error(`error status: ${status}`);
}
token = data.user.token
}
await nonOurbigbookOptions.sequelize.transaction(async (transaction) => {
await nonOurbigbookOptions.sequelize.models.Cli.update(
{
defaultUsernameForHost: false
},
{
where: {
host,
},
transaction,
}
)
await nonOurbigbookOptions.sequelize.models.Cli.upsert(
{
host,
username,
token,
// Use the latest one by default.
defaultUsernameForHost: true
},
{ transaction }
)
})
}
let currentBuildToken
if (cli.webCancelBuild) {
const current = await webApi.articlesCurrentBuild({ timeout: 15000 })
assertApiStatus(current.status, current.data)
const result = await webApi.articlesCancelBuild(current.data.build?.token || null, undefined, { timeout: 15000 })
if (result.status !== 202) assertApiStatus(result.status, result.data)
console.log(`web_upload: @${username}: ${result.data.message}`)
process.exit(0)
}
if (cli.webWatch) {
await watchWebBuild(webApi, username)
process.exit(0)
}
if (!cli.webDry && !cli.webIndividualUpload && !cli.webNestedSet) {
const { data, status } = await webApi.articlesCurrentBuild({ timeout: 15000 })
assertApiStatus(status, data)
const ongoing = data.job || data.tree || (data.build && !['completed', 'failed'].includes(data.build.status))
if (ongoing && !cli.webForce) {
let answer = ''
if (process.stdin.isTTY) {
const [error, response] = await read({ prompt: 'A web build exists. Discard it and start a new upload? [y/N] ' })
if (error) throw error
answer = response
} else {
console.log(`A web build exists; ${cli.webNoWatch ? 'leaving it running' : 'watching it'}. Use --web-force to replace it non-interactively.`)
}
if (!/^y(es)?$/i.test(answer.trim())) {
if (!cli.webNoWatch) await watchWebBuild(webApi, username)
process.exit(0)
}
}
currentBuildToken = require('crypto').randomBytes(16).toString('hex')
while (true) {
const result = await webApi.articlesBuildReplace(currentBuildToken, data.build?.token || null, { timeout: 15000 })
if (result.status !== 202) assertApiStatus(result.status, result.data)
if (result.data.build.status !== 'cancelling') break
console.log('web_upload: waiting for the current article/tree transaction to finish before replacement')
await new Promise(resolve => setTimeout(resolve, 1000))
}
}
if (cli.webNestedSet) {
await updateNestedSet(webApi, username, { foreground: cli.webIndividualUpload, watch: !cli.webNoWatch })
process.exit(0)
}
// Do a local conversion that splits mutiheader files into single header files for upload.
ourbigbook_options.split_headers = true
ourbigbook_options.render_include = false
ourbigbook_options.forbid_multi_h1 = true
// We create this quick and dirty separate database to store information for upload.
// Technically much of this information is part of Article, but factoring that would be risky/hard,
// it is not worth it.
//
// Adding this cache because I had an unminimizable error on the main document, and we have to save some time
// or else I can't minimize it, this way we can skip the initial bigb split render conversion and go
// straight to upload.
const webCache = await createWebCache(nonOurbigbookOptions.outdir)
const sequelizeWeb = webCache.sequelize
const sequelizeWebArticle = sequelizeWeb.models.Article
const sequelizeWebIndexId = sequelizeWeb.models.IndexId
nonOurbigbookOptions.post_convert_callback = webCache.postConvert
let treeToplevelId, treeToplevelFileId
if (input_path_is_file) {
await convert_path_to_file(inputPath, ourbigbook_options, nonOurbigbookOptions)
treeToplevelFile = await nonOurbigbookOptions.sequelize.models.File.findOne({
where: { path: inputPath } })
treeToplevelFileId = treeToplevelFile.id
treeToplevelId = treeToplevelFile.toplevel_id
} else {
// TODO non toplevel directory not supported yet.
await convert_directory_extract_ids_and_render(
inputPath,
ourbigbook_options,
nonOurbigbookOptions,
)
const index = (await sequelizeWebIndexId.findAll())[0]
if (index === undefined) {
cli_error('a toplevel index is mandatory for web uploads')
}
treeToplevelId = index.idid
}
if (nonOurbigbookOptions.had_error) {
process.exit(1)
}
const upload = async () => {
let header_tree = []
if (input_path_is_file || cli.webId) {
let toPush
if (cli.webId) {
toPush = cli.webId
} else {
toPush = treeToplevelId
}
header_tree.push({ to_id: toPush })
} else {
// Fake an index entry at the end so that the index will get rendered.
// It is not otherwise present as it has no parents.
header_tree.push({ to_id: '' })
}
if (!cli.webId) {
header_tree = header_tree.concat(await ourbigbook_options.db_provider.fetch_header_tree_ids(
[treeToplevelId],
{
definedAtFileId: treeToplevelFileId,
}
))
}
// A single-file or single-ID upload may not fetch the surrounding header
// tree. Fetch missing parent and previous-sibling metadata explicitly so
// it still uses the authoritative cross-file Ref graph rather than the
// lossy split-file web cache.
if (input_path_is_file || cli.webId) {
const { Ref } = nonOurbigbookOptions.sequelize.models
for (const headerTreeEntry of header_tree) {
if (headerTreeEntry.from_id === undefined && headerTreeEntry.to_id !== '') {
const parentRef = await Ref.findOne({
attributes: ['from_id', 'to_id_index'],
raw: true,
where: {
to_id: headerTreeEntry.to_id,
type: Ref.Types[ourbigbook.REFS_TABLE_PARENT],
},
})
if (parentRef) {
headerTreeEntry.from_id = parentRef.from_id
if (parentRef.to_id_index > 0) {
const previousSiblingRef = await Ref.findOne({
attributes: ['to_id'],
order: [['to_id_index', 'DESC']],
raw: true,
where: {
from_id: parentRef.from_id,
to_id_index: { [Op.lt]: parentRef.to_id_index },
type: Ref.Types[ourbigbook.REFS_TABLE_PARENT],
},
})
if (previousSiblingRef) {
headerTreeEntry.previous_sibling_id = previousSiblingRef.to_id
}
}
}
}
}
}
let webStartIndex = 0
if (cli.webStartId !== undefined) {
webStartIndex = header_tree.findIndex(entry => entry.to_id === cli.webStartId)
if (webStartIndex === -1) {
cli_error(`--web-start-id ID not found in the upload tree: "${cli.webStartId}"`)
}
}
const dorender = [false]
if (!cli.webIndividualUpload) dorender.push('check')
if (cli.webRender) {
dorender.push(true)
}
let data, status, i = 0
const webPathToArticle = {}
if (!cli.webDry) {
do {
;({ data, status } = await webApi.articlesHash({ author: username, offset: i }))
assertApiStatus(status, data)
const articles = data.articles
for (const article of articles) {
webPathToArticle[article.path] = article
}
i += articles.length
} while (data.articles.length === ARTICLE_HASH_LIMIT_MAX)
}
const idToArticleMeta = {}
const localArticles = await sequelizeWebArticle.findAll({ attributes: ['source', 'title', 'idid', 'inpath', 'parentId'] })
for (const article of localArticles) {
idToArticleMeta[article.idid] = article
}
// The split-file cache cannot represent a toplevel header's cross-file
// Include parent. Derive all upload tree metadata from the authoritative
// Ref graph returned in header_tree. Calculate it before applying
// --web-start-id so resumed uploads can replay skipped prerequisites.
const idToWebTreeMeta = {}
const lastChildIdByParentId = new Map()
for (const header_tree_entry of header_tree) {
const articleMeta = idToArticleMeta[header_tree_entry.to_id]
// Can fail for synonyms.
if (articleMeta) {
const parentId = header_tree_entry.from_id === undefined
// The synthetic index entry, and a single file whose parent is not
// present in the local database, have no Ref metadata to override.
? articleMeta.parentId
: header_tree_entry.from_id
let previousSiblingId = header_tree_entry.previous_sibling_id
if (previousSiblingId === undefined && parentId !== null && parentId !== undefined) {
previousSiblingId = lastChildIdByParentId.get(parentId)
}
if (parentId !== null && parentId !== undefined) {
lastChildIdByParentId.set(parentId, header_tree_entry.to_id)
}
idToWebTreeMeta[header_tree_entry.to_id] = {
parentId,
previousSiblingId,
}
if (cli.addTestInstrumentation) {
console.error(
`web_upload_tree: ${JSON.stringify(header_tree_entry.to_id)} parent=${JSON.stringify(parentId)} previous=${JSON.stringify(previousSiblingId)}`
)
}
}
}
// A suffix is not necessarily self-contained because fetch_header_tree_ids
// returns breadth-first order. A child after the resume marker can therefore
// have a parent before it. Replay skipped parents and previous siblings (and
// their own prerequisites) so resuming also works when those server rows are
// absent, while still avoiding unrelated completed subtrees.
const webResumeDependencyIds = new Set()
if (webStartIndex > 0) {
const headerTreeIdToIndex = new Map(header_tree.map(
(entry, index) => [entry.to_id, index]
))
const pendingIds = header_tree.slice(webStartIndex).map(entry => entry.to_id)
for (let pendingIndex = 0; pendingIndex < pendingIds.length; pendingIndex++) {
const webTreeMeta = idToWebTreeMeta[pendingIds[pendingIndex]]
if (!webTreeMeta) {
continue
}
for (const dependencyId of [webTreeMeta.parentId, webTreeMeta.previousSiblingId]) {
const dependencyIndex = headerTreeIdToIndex.get(dependencyId)
if (
dependencyIndex !== undefined &&
dependencyIndex < webStartIndex &&
!webResumeDependencyIds.has(dependencyId)
) {
webResumeDependencyIds.add(dependencyId)
pendingIds.push(dependencyId)
}
}
}
}
let nestedSetNeedsUpdate = false
// Partial uploads must not delete unrelated server articles or files.
const fullWebUpload = inputPath === '.' && !cli.webId
if (fullWebUpload || localMediaDir) {
// Handle uploads
{
const uploadPaths = new Map(fullWebUpload
? [...nonOurbigbookOptions.nonBigbPathsConverted].map(p => [p, p]) : [])
if (localMediaDir) {
const ignored = git_is_in_repo(localMediaDir)
? new Set(runCmd('git', ['-C', localMediaDir, 'ls-files', '--ignored', '--others', '--exclude-standard', '--directory', '-z'], cmdOptsInfo)
.split('\0').filter(Boolean).map(p => p.replace(/\/$/, '')))
: new Set()
// Walk this repository separately: source discovery deliberately skips -/.
for (const file of walk_directory_recursively(localMediaDir,
new Set([...DEFAULT_IGNORE_BASENAMES_SET, '.gitignore', '.gitattributes', '.gitmodules']),
ignored, [], [], localMediaDir)) {
const stat = fs.lstatSync(file)
const destination = path.relative(localMediaDir, file)
const sourcePath = path.join(ourbigbook_json_dir, destination)
// Validate the source tree too, including partial uploads and
// article sources that are not in the ordinary upload manifest.
// Shared directories are fine; files must never overwrite either.
if (uploadPaths.has(destination) ||
(fs.existsSync(sourcePath) &&
!(stat.isDirectory() && fs.lstatSync(sourcePath).isDirectory()))) {
cli_error(`media upload path collision: ${destination}`)
}
if (!stat.isFile()) continue
uploadPaths.set(destination, file)
}
}
const uploadHashes = {}
const serverOnlyUploads = []
let i = 0
if (!cli.webDry) {
do {
;({ data, status } = await webApi.uploadHash({ author: username, offset: i }))
assertApiStatus(status, data)
const uploads = data.uploads
for (const upload of uploads) {
const pathNoUsername = upload.path.substring(username.length + URL_SEP.length)
uploadHashes[pathNoUsername] = upload.hash
if (fullWebUpload && !uploadPaths.has(pathNoUsername) && upload.list !== false) {
serverOnlyUploads.push(pathNoUsername)
}
}
i += uploads.length
} while (data.uploads.length === ARTICLE_HASH_LIMIT_MAX)
}
i = 0
for (const p of serverOnlyUploads) {
// Preserve existing raw URLs and bytes; removal from listings is
// allowed for owners, unlike the admin-only deletion endpoint.
await webCreateOrUpdate({
cleanupDeleted: true,
fn: async () => webApi.uploadUpdate(`${username}${URL_SEP}${p}`, { list: false }),
i: i++,
isUpload: true,
title: p,
webDry: cli.webDry,
})
}
for (const [p, file] of uploadPaths) {
const content = fs.readFileSync(file)
if (hashToHex(content) !== uploadHashes[p]) {
await webCreateOrUpdate({
cleanupDeleted: false,
fn: async () => webApi.uploadCreateOrUpdate(`${username}${URL_SEP}${p}`, content),
i: i++,
isUpload: true,
title: p,
webDry: cli.webDry,
})
}
}
}
}
if (fullWebUpload) {
// https://docs.ourbigbook.com/todo/make-articles-removed-locally-empty-on-web-upload
const serverOnlyPaths = new Set(Object.keys(webPathToArticle))
for (const header_tree_entry of header_tree) {
serverOnlyPaths.delete(pathUsernameAndExt(username, header_tree_entry.to_id))
}
let i = 0
for (const path of serverOnlyPaths) {
if (webPathToArticle[path].cleanupIfDeleted) {
const render = true
const ret = await webCreateOrUpdate({
cleanupDeleted: true,
fn: async () => webApi.articleCreateOrUpdate(
{
bodySource: '',
},
{
path: pathNoUsernameNoext(path),
render,
list: false,
updateNestedSetIndex: !cli.webNestedSetBulk,
},
),
i: i++,
inpath: path,
render,
webDry: cli.webDry,
})
if (ret.nestedSetNeedsUpdate) {
nestedSetNeedsUpdate = true
}
}
}
}
let buildId = currentBuildToken
const buildJobs = []
for (const render of dorender) {
let i = 0
let renderBatch = []
let batchSourceIds = []
const phase = !render ? 'extract' : render === 'check' ? 'check' : 'render'
const phaseBatches = []
const flushRenderBatch = async () => {
if (renderBatch.length) {
if (cli.webDry) console.log(`web_${phase}: would queue ${renderBatch.length} articles`)
else {
// Plan exact count- and byte-limited batches before staging so
// progress has a real denominator. Keep source bodies on disk.
phaseBatches.push({
targets: renderBatch.map(({ article, ...target }) => target),
sourceIds: batchSourceIds,
})
}
renderBatch = []
batchSourceIds = []
}
}
// This ordering ensures parents come before children.
for (const [headerTreeIndex, header_tree_entry] of header_tree.entries()) {
const id = header_tree_entry.to_id
const articleMeta = idToArticleMeta[id]
const webTreeMeta = idToWebTreeMeta[id]
if (
// Can fail for synonyms.
articleMeta && webTreeMeta
) {
// Prerequisites only need their ID extraction replayed. Rendering
// still begins exactly at the requested marker.
if (
headerTreeIndex < webStartIndex &&
(render || !webResumeDependencyIds.has(id))
) {
continue
}
const inpathParse = path.parse(articleMeta.inpath)
const articlePath = path.join(inpathParse.dir, inpathParse.name)
const webPathToArticleEntry = webPathToArticle[pathUsernameAndExt(username, articlePath)]
if (webPathToArticleEntry === undefined ) {
webPathToArticleEntry
}
const articleHashProps = {
source: articleMeta.source,
list: true,
}
const parentId = addUsername(webTreeMeta.parentId, username)
if (parentId !== null) {
articleHashProps.parentId = parentId
}
const previousSiblingId = addUsername(webTreeMeta.previousSiblingId, username)
if (previousSiblingId !== null) {
articleHashProps.previousSiblingId = previousSiblingId
}
// A completed preflight survives worker/client restarts even when
// rendering is still outstanding. Render itself always validates
// against the current database; this only skips the separate pass.
if (
render === 'check' &&
!cli.webForceIdExtraction &&
webPathToArticleEntry &&
webPathToArticleEntry.hash === articleHash(articleHashProps) &&
webPathToArticleEntry.checkedHash === webPathToArticleEntry.hash
) continue
if (
webPathToArticleEntry === undefined ||
webPathToArticleEntry.hash !== articleHash(articleHashProps) ||
(
render &&
(
webPathToArticleEntry.renderOutdated ||
cli.webForceRender ||
(render === 'check' && cli.webForceIdExtraction)
)
) ||
(
!render &&
(
cli.webForceIdExtraction ||
webResumeDependencyIds.has(id)
)
)
) {
// OK, we are going to render this article, so fetch it fully now including the source.
const article = await sequelizeWebArticle.findOne({ where: { idid: id } })
let data, status
const articleArgs = {}
if (!render) {
articleArgs.titleSource = article.title
articleArgs.bodySource = article.body
}
const extraArgs = {
path: articlePath,
render: Boolean(render),
}
if (parentId) {
extraArgs.parentId = parentId
}
if (previousSiblingId) {
extraArgs.previousSiblingId = previousSiblingId
}
extraArgs.updateNestedSetIndex = !cli.webNestedSetBulk
extraArgs.list = true
if (!cli.webIndividualUpload) {
const target = { ...extraArgs, ...(render ? { hash: articleHash(articleHashProps) } : { article: articleArgs }) }
if (renderBatch.length === ARTICLE_RENDER_BATCH_LIMIT || Buffer.byteLength(JSON.stringify([...renderBatch, target])) > 4 * 1024 * 1024) await flushRenderBatch()
renderBatch.push(target)
if (!render) batchSourceIds.push(id)
i++
} else {
const ret = await webCreateOrUpdate({
fn: async () => webApi.articleCreateOrUpdate(articleArgs, extraArgs),
i: i++,
inpath: article.inpath,
render,
title: article.title,
webDry: cli.webDry,
})
if (render && ret.nestedSetNeedsUpdate) {
nestedSetNeedsUpdate = true
}
}
if (
render &&
cli.webMaxRenders !== undefined &&
i === cli.webMaxRenders
) {
break
}
}
}
}
await flushRenderBatch()
for (const [batchIndex, batch] of phaseBatches.entries()) {
if (!buildId) {
buildId = require('crypto').randomBytes(16).toString('hex')
const { data, status } = await webApi.articlesBuildReplace(buildId, null, { timeout: 20000 })
checkWebBulkBusy(status, data)
if ([404, 405].includes(status)) console.error('Durable background uploads require an updated server. Use --web-individual-upload for the legacy path.')
if (status !== 202) assertApiStatus(status, data)
}
const sources = !render && new Map((await sequelizeWebArticle.findAll({
attributes: ['idid', 'title', 'body'], where: { idid: { [Op.in]: batch.sourceIds } },
})).map(article => [article.idid, article]))
const targets = render ? batch.targets : batch.targets.map((target, index) => {
const article = sources.get(batch.sourceIds[index])
return { ...target, article: { titleSource: article.title, bodySource: article.body } }
})
const id = await queueWebArticles(webApi, targets, {
phase, start: false, batchIndex, batchCount: phaseBatches.length, buildId, buildIndex: buildJobs.length,
})
buildJobs.push({ id, phase, batchIndex, batchCount: phaseBatches.length, firstArticle: targets[0].path })
}
}
if (buildId) {
let rebuildTree = Boolean(cli.webNestedSetBulk && (buildJobs.length || nestedSetNeedsUpdate))
if (cli.webNestedSetBulk && !rebuildTree) {
const result = await webApi.user(username)
assertApiStatus(result.status, result.data)
rebuildTree = Boolean(result.data.nestedSetNeedsUpdate)
}
const { data, status } = await webApi.articlesCurrentBuildCommit(buildId, buildJobs.length, rebuildTree, { timeout: 20000 })
checkWebBulkBusy(status, data)
if (status !== 202) assertApiStatus(status, data)
console.log(`Web upload has submission complete (${buildJobs.length} jobs). The server can now finish by itself even if you close the CLI.`)
if (!cli.webNoWatch) {
for (const job of buildJobs) await waitWebArticles(webApi, job.id, job.phase, job)
await waitWebBuild(webApi, buildId)
}
}
if (!buildId && cli.webNestedSetBulk && !cli.webDry) {
if (!nestedSetNeedsUpdate) {
;({ data, status } = await webApi.user(username))
assertApiStatus(status, data)
if (data.nestedSetNeedsUpdate) {
nestedSetNeedsUpdate = true
}
}
if (nestedSetNeedsUpdate) {
await updateNestedSet(webApi, username, { foreground: cli.webIndividualUpload })
}
}
}
await upload()
await sequelizeWeb.close()
} else if (cli.watch) {
if (cli.stdout) {
cli_error('--stdout and --watch are incompatible');
}
if (publish) {
cli_error('--publish and --watch are incompatible');
}
if (cli.viewOutput) {
cli_error('--view-output and --watch are incompatible');
}
await create_db(ourbigbook_options, nonOurbigbookOptions);
if (!input_path_is_file) {
await reconcile_db_and_filesystem(inputPath, ourbigbook_options, nonOurbigbookOptions);
await convert_directory_extract_ids(inputPath, ourbigbook_options, nonOurbigbookOptions);
}
const watcher = require('chokidar').watch(inputPath, {ignored: DEFAULT_IGNORE_BASENAMES})
const convert = async (subpath) => {
await convert_path_to_file(subpath, ourbigbook_options, nonOurbigbookOptions);
await check_db(nonOurbigbookOptions)
nonOurbigbookOptions.ourbigbook_paths_converted = []
}
watcher.on('change', convert).on('add', convert)
} else {
if (input_path_is_file) {
if (publish) {
cli_error('--publish must take a directory as input, not a file');
}
await create_db(ourbigbook_options, nonOurbigbookOptions);
for (const inputPath of inputPaths) {
if (ignore_path(
DEFAULT_IGNORE_BASENAMES_SET,
nonOurbigbookOptions.ignore_paths,
nonOurbigbookOptions.ignore_path_regexps,
nonOurbigbookOptions.dont_ignore_path_regexps,
inputPath
)) {
console.error(`skipping conversion of "${inputPath}" because it is ignored`)
} else {
output = await convert_path_to_file(inputPath, ourbigbook_options, nonOurbigbookOptions);
}
}
if (!nonOurbigbookOptions.had_error && cli.render) {
await check_db(nonOurbigbookOptions)
}
if (
cli.viewOutput &&
!nonOurbigbookOptions.had_error
) {
const output_path = nonOurbigbookOptions.last_output_path
if (output_path === undefined) {
cli_error('could not determine output file for --view-output')
}
viewOutput(output_path, cmdOpts)
}
} else {
if (cli.stdout) {
cli_error('--publish cannot be used in directory conversion');
}
let actualInputDir;
let publishBranch;
let publishOutPublishDir;
let publishOutPublishDirCwd;
let publishOutPublishDistDir;
let publishRemoteUrl;
let srcBranch;
if (publish) {
if (localMediaDir && publish_uses_git) {
const mediaGit = args => chomp(runCmd('git', ['-C', localMediaDir, ...args], cmdOptsInfo))
if (fs.realpathSync(mediaGit(['rev-parse', '--show-toplevel'])) !== fs.realpathSync(localMediaDir)) {
cli_error('local media path must be the root of its own Git repository for GitHub publishing')
}
const remote = require('git-url-parse')(mediaGit(['remote', 'get-url', 'origin']))
if (remote.source !== 'github.com') cli_error('local media repository origin must be on github.com for GitHub publishing')
const commit = mediaGit(['rev-parse', 'HEAD'])
let branch = mediaGit(['rev-parse', '--abbrev-ref', 'HEAD'])
// A submodule can pin an older, already published commit. Do not try
// to rewind its remote branch to that commit.
const alreadyPublished = branch === 'HEAD' && mediaGit([
'for-each-ref', '--contains', commit, '--format=%(refname)', 'refs/remotes/origin',
]) !== ''
if (branch === 'HEAD' && !alreadyPublished) {
branch = mediaGit(['symbolic-ref', '--short', 'refs/remotes/origin/HEAD']).replace(/^origin\//, '')
}
ourbigbook_options.localMediaPublish = true
const snapshot = fs.mkdtempSync(path.join(require('os').tmpdir(), 'ourbigbook-publish-media-'))
process.on('exit', () => fs.rmSync(snapshot, { recursive: true, force: true }))
runCmd('git', ['clone', '--shared', '--no-checkout', '--', localMediaDir, snapshot], cmdOptsNoDry)
runCmd('git', ['-C', snapshot, 'checkout', '--detach', commit], cmdOptsNoDry)
Object.assign(nonOurbigbookOptions, { localMediaPath, localMediaSnapshot: snapshot })
Object.assign(ourbigbook_options, fileConversionOptions(ourbigbook_json_dir, nonOurbigbookOptions))
// Publish media first, including before cloning a source repository with submodules.
if (!cli.dryRunPush && !alreadyPublished) runCmd('git', ['-C', localMediaDir, 'push', 'origin', `HEAD:refs/heads/${branch}`], cmdOpts)
}
// Clone the source to ensure that only git tracked changes get built and published.
ourbigbook_options.template_vars.publishTargetIsWebsite = publishTargetIsWebsite
if (!isInGitRepo) {
cli_error('--publish must point to a path inside a git repository');
}
if (publish_uses_git) {
// TODO ideally we should use the default remote for the given current branch, but there doesn't seem
// to be a super easy way for now, so we just hardcode origin to start with.
// https://stackoverflow.com/questions/171550/find-out-which-remote-branch-a-local-branch-is-tracking
const opts = {}
const originUrl = chomp(runCmd('git', ['-C', inputPathCwd, 'config', '--get', 'remote.origin.url'], cmdOptsInfoNothrow))
if (cmdOptsInfoNothrow.extra_returns.out.status != 0) {
cli_error('a "origin" git remote repository is required to publish, configure it with something like "git remote add origin git@github.com:username/reponame.git"')
}
if (options.ourbigbook_json.publishRemoteUrl) {
publishRemoteUrl = options.ourbigbook_json.publishRemoteUrl
} else {
publishRemoteUrl = originUrl
}
if (!publishRemoteUrl) {
publishRemoteUrl = 'git@github.com:ourbigbook/ourbigbook.git';
}
srcBranch = chomp(runCmd('git', ['-C', inputPathCwd, 'rev-parse', '--abbrev-ref', 'HEAD'], cmdOptsInfo))
const parsed_remote_url = require("git-url-parse")(publishRemoteUrl);
if (parsed_remote_url.source !== 'github.com') {
cli_error('only know how to publish to origin == github.com currently, please send a patch');
}
let remote_url_path_components = parsed_remote_url.pathname.split(path.sep);
if (options.ourbigbook_json.publishBranch) {
publishBranch = options.ourbigbook_json.publishBranch
} else if (publish_target === 'github-md') {
publishBranch = 'master'
} else if (remote_url_path_components[2].startsWith(remote_url_path_components[1] + '.github.io')) {
publishBranch = 'master';
} else {
publishBranch = 'gh-pages';
}
if (
publishRemoteUrl === originUrl &&
srcBranch === publishBranch
) {
cli_error(`source and publish branches are the same: ${publishBranch}`);
}
}
fs.mkdirSync(publishDir, { recursive: true });
if (cli.publishCommit !== undefined) {
runCmd('git', ['-C', inputPathCwd, 'add', '-u'], cmdOpts);
runCmd( 'git', ['-C', inputPathCwd, 'commit', '-m', cli.publishCommit], cmdOpts);
}
sourceCommit = gitSha(inputPath, srcBranch);
if (fs.existsSync(publish_git_dir)) {
runCmd('git', ['-C', publishDirCwd, 'checkout', '--', '.'], cmdOptsNoDry);
runCmd('git', ['-C', publishDirCwd, 'fetch'], cmdOptsNoDry);
runCmd('git', ['-C', publishDirCwd, 'checkout', sourceCommit], cmdOptsNoDry);
runCmd('git', ['-C', publishDirCwd, 'submodule', 'update', '--init'], cmdOptsNoDry);
runCmd('git', ['-C', publishDirCwd, 'clean', '-xdf'], cmdOptsNoDry);
} else {
runCmd('git', ['clone', '--recursive', '--depth', '1', input_git_toplevel, publishDirCwd],
ourbigbook.cloneAndSet(cmdOpts, 'dry_run', false));
}
// Set some variables especially for publishing.
actualInputDir = path.join(publishDir, subdir_relpath);
nonOurbigbookOptions.ourbigbook_json_dir = actualInputDir;
publishOutPublishDir = path.join(publish_tmpdir, publish_target);
publishOutPublishDirCwd = relpathCwd(publishOutPublishDir)
publish_out_publish_obb_dir = path.join(publishOutPublishDir, ourbigbook_nodejs.PUBLISH_OBB_PREFIX)
publishOutPublishDistDir = path.join(publishOutPublishDir, ourbigbook_nodejs.PUBLISH_ASSET_DIST_PREFIX)
nonOurbigbookOptions.out_css_path = path.join(publishOutPublishDistDir, ourbigbook_nodejs.DIST_CSS_BASENAME);
nonOurbigbookOptions.out_js_path = path.join(publishOutPublishDistDir, ourbigbook_nodejs.DIST_JS_BASENAME);
nonOurbigbookOptions.external_css_and_js = true;
// Remove all files from the publish diretory in case some were removed from the original source.
if (!cli.publishNoConvert) {
if (publish_uses_git) {
if (fs.existsSync(path.join(publishOutPublishDir, '.git'))) {
// Quiet avoids overflowing spawnSync's output buffer on large sites.
// Empty indexes are OK; actual cleanup failures must abort the build.
// Dry runs also rebuild locally, so they need the same clean output.
runCmd('git', ['-C', publishOutPublishDirCwd, 'rm', '--quiet', '-r', '-f', '--ignore-unmatch', '--', '.'], cmdOptsNoDry)
// An interrupted publish can also leave untracked/ignored output.
runCmd('git', ['-C', publishOutPublishDirCwd, 'clean', '--quiet', '-fdx'], cmdOptsNoDry)
}
} else {
fs.rmSync(publishOutPublishDir, { recursive: true, force: true })
}
// Clean database to ensure a clean conversion. TODO: this is dangerous, if some day we
// start adding more conversion state outside of db.sqlite3. Better would be to remove the
// entire _out/publish/_out. Te slight downside of that is that:
// - it deletes other publish targets
// - it forces re-fetch of git history on the gh-pages branch
fs.rmSync(path.join(publish_tmpdir, ourbigbook_nodejs_front.SQLITE_DB_BASENAME), { force: true })
fs.mkdirSync(publishOutPublishDir, { recursive: true })
}
} else {
actualInputDir = inputPath;
publishOutPublishDir = outdir;
publishOutPublishDirCwd = relpathCwd(publishOutPublishDir)
}
nonOurbigbookOptions.outdir = publishOutPublishDir;
if (!cli.publishNoConvert) {
await create_db(ourbigbook_options, nonOurbigbookOptions);
// Do the actual conversion.
await convert_directory_extract_ids_and_render(actualInputDir, ourbigbook_options, nonOurbigbookOptions)
if (nonOurbigbookOptions.had_error) {
process.exit(1);
}
// Generate redirects from ourbigbook.json.
if (output_format === ourbigbook.OUTPUT_FORMAT_HTML) {
for (let [from, to] of options.ourbigbook_json.redirects) {
if (
// TODO https://docs.ourbigbook.com/respect-ourbigbook-json-htmlxextension-on-ourbigbook-json-redirects
ourbigbook_options.htmlXExtension === false ? false : true &&
!ourbigbook.protocolIsKnown(to)
) {
to += '.' + ourbigbook.HTML_EXT
}
generate_redirect_base(
path.join(nonOurbigbookOptions.outdir, from + '.' + ourbigbook.HTML_EXT),
to
)
}
}
}
// Publish the converted output if build succeeded.
if (
publish &&
!nonOurbigbookOptions.had_error
) {
// Push the original source.
if (publishIsPublic && !cli.dryRunPush) {
runCmd('git', ['-C', inputPathCwd, 'push'], cmdOpts);
}
if (publish_uses_git) {
runCmd('git', ['-C', publishOutPublishDirCwd, 'init'], cmdOpts);
const coreSshCommand = chomp(runCmd('git', ['-C', inputPath, 'config', '--get', 'core.sshCommand'], cmdOptsInfoNothrow))
if (coreSshCommand) {
runCmd('git', ['-C', publishOutPublishDirCwd, 'config', 'core.sshCommand', coreSshCommand], cmdOpts)
}
// https://stackoverflow.com/questions/42871542/how-to-create-a-git-repository-with-the-default-branch-name-other-than-master
runCmd('git', ['-C', publishOutPublishDirCwd, 'checkout', '-B', publishBranch], cmdOptsNoStdout);
try {
// Fails if remote already exists.
runCmd('git', ['-C', publishOutPublishDirCwd, 'remote', 'add', 'origin', publishRemoteUrl], cmdOpts);
} catch(error) {
runCmd('git', ['-C', publishOutPublishDirCwd, 'remote', 'set-url', 'origin', publishRemoteUrl], cmdOpts);
}
// Ensure that we are up-to-date with the upstream gh-pages if one exists.
runCmd('git', ['-C', publishOutPublishDirCwd, 'fetch', 'origin'], cmdOpts);
runCmd(
'git',
['-C', publishOutPublishDirCwd, 'reset', '--quiet', `origin/${publishBranch}`],
// Fails on the first commit in an empty repository.
ourbigbook.cloneAndSet(cmdOpts, 'throwOnError', false)
);
}
// Generate special files needed for a given publish target.
for (const p in publish_create_files) {
const outpath = path.join(publishOutPublishDir, p)
fs.mkdirSync(path.dirname(outpath), { recursive: true });
fs.writeFileSync(outpath, publish_create_files[p])
}
if (options.ourbigbook_json.prepublish) {
if (!cli.dryRun && !cli.dryRunPush && !cli.unsafeAce) {
cli_error('prepublish in ourbigbook.json requires running with --unsafe-ace');
}
const prepublish_path = options.ourbigbook_json.prepublish
if (!fs.existsSync(prepublish_path)) {
cli_error(`${ourbigbook.OURBIGBOOK_JSON_BASENAME} prepublish file not found: ${prepublish_path}`);
}
try {
runCmd('./' + path.relative(process.cwd(), path.resolve(prepublish_path)), [relpathCwd(publishOutPublishDir)]);
} catch(error) {
cli_error(`${ourbigbook.OURBIGBOOK_JSON_BASENAME} prepublish command exited non-zero, aborting`);
}
}
if (output_format === ourbigbook.OUTPUT_FORMAT_HTML) {
fs.mkdirSync(publish_out_publish_obb_dir, { recursive: true })
if (
options.ourbigbook_json.generateSitemap &&
options.ourbigbook_json.publishRootUrl
) {
let sitemap = nonOurbigbookOptions.sitemap
if (!nonOurbigbookOptions.options.htmlXExtension) {
for (let i = 0; i < sitemap.length; i++) {
let p = sitemap[i]
let pSplit = p.split(URL_SEP)
if (pSplit[pSplit.length - 1] === 'index.html') {
p = pSplit.slice(0, -1).join(URL_SEP)
}
if (p.endsWith('.' + ourbigbook.HTML_EXT)) {
p = p.slice(0, - (ourbigbook.HTML_EXT.length + 1))
}
sitemap[i] = p
}
}
const sitemapPath = path.join(publish_out_publish_obb_dir, 'sitemap.txt')
console.log(`sitemap: ${relpathCwd(sitemapPath)}`)
fs.writeFileSync(
sitemapPath,
sitemap
// Not ideal to remove duplicates here, but there were some
// duplicate _file cases that would require some thought.
.sort()
.filter((elem, index, arr) =>
index === arr.length - 1 || arr[index + 1] !== elem
).map(p =>
`${options.ourbigbook_json.publishRootUrl}${p ? URL_SEP : ''}${p}`
).join('\n')
)
}
// Copy runtime assets from -/obb/ into the output repository.
const dir = fs.opendirSync(ourbigbook_nodejs.DIST_PATH)
let dirent
while ((dirent = dir.readSync()) !== null) {
require('fs-extra').copySync(
path.join(ourbigbook_nodejs.DIST_PATH, dirent.name),
path.join(publishOutPublishDistDir, dirent.name)
)
}
if (options.ourbigbook_json.web && options.ourbigbook_json.web.linkFromStaticHeaderMetaToWeb) {
fs.copyFileSync(
ourbigbook_nodejs.LOGO_PATH,
path.join(publishOutPublishDir, ourbigbook_nodejs.LOGO_ROOT_RELPATH)
)
}
dir.closeSync()
}
if (publish_uses_git) {
if (nonOurbigbookOptions.localMediaSnapshot) {
// Merge only the committed snapshot, never dirty/untracked files or Git metadata.
const copyMedia = (validateOnly, relative='') => {
for (const entry of fs.readdirSync(path.join(nonOurbigbookOptions.localMediaSnapshot, relative), { withFileTypes: true })) {
if (['.git', '.gitignore', '.gitattributes', '.gitmodules'].includes(entry.name)) continue
const mediaPath = path.join(relative, entry.name)
const destination = path.join(publishOutPublishDir, mediaPath)
if (entry.isSymbolicLink()) cli_error(`cannot publish local media symbolic link: ${mediaPath}`)
if (entry.isDirectory()) {
if (fs.existsSync(destination) && !fs.lstatSync(destination).isDirectory()) {
cli_error(`local media collides with generated output: ${mediaPath}`)
}
if (!validateOnly) fs.mkdirSync(destination, { recursive: true })
copyMedia(validateOnly, mediaPath)
} else {
if (fs.existsSync(destination)) cli_error(`local media collides with generated output: ${mediaPath}`)
if (!validateOnly) fs.copyFileSync(path.join(nonOurbigbookOptions.localMediaSnapshot, mediaPath), destination, fs.constants.COPYFILE_EXCL)
}
}
}
// Detect every collision before copying any media into the output.
copyMedia(true)
copyMedia(false)
}
// Commit and push.
runCmd('git', ['-C', publishOutPublishDirCwd, 'add', '.'], cmdOpts);
const args = ['-C', publishOutPublishDirCwd, 'commit', '-m', sourceCommit]
if (git_has_commit(publishOutPublishDir)) {
args.push('--amend')
}
const commitCmdOptions = { ...cmdOptsNoStdout }
const name = chomp(runCmd('git', ['-C', inputPath, 'config', '--get', 'user.name'], cmdOpts))
const email = chomp(runCmd('git', ['-C', inputPath, 'config', '--get', 'user.email'], cmdOpts))
if (name && email) {
commitCmdOptions.env_extra = {
...commitCmdOptions.env_extra,
...{
GIT_COMMITTER_EMAIL: email,
GIT_COMMITTER_NAME: name,
GIT_AUTHOR_EMAIL: email,
GIT_AUTHOR_NAME: name,
},
}
args.push(...['--author', `${name} <${email}>`])
}
if (options.ourbigbook_json.publishCommitDate) {
args.push(...['--date', options.ourbigbook_json.publishCommitDate])
Object.assign(commitCmdOptions.env_extra, { GIT_COMMITTER_DATE: options.ourbigbook_json.publishCommitDate })
}
runCmd('git', args, commitCmdOptions);
if (!cli.dryRunPush) {
runCmd('git', ['-C', publishOutPublishDirCwd, 'push', '-f', 'origin', `${publishBranch}:${publishBranch}`], cmdOpts);
// Mark the commit with the `published` branch to make it easier to find what was last published.
runCmd('git', ['-C', inputPathCwd, 'checkout', '-B', 'published'], cmdOpts);
runCmd('git', ['-C', inputPathCwd, 'push', '-f', '--follow-tags'], cmdOpts);
runCmd('git', ['-C', inputPathCwd, 'checkout', '-'], cmdOpts);
}
}
}
}
}
}
if (
// Happens on empty input from stdin (Ctrl + D withotu typing anything)
output !== undefined &&
(
inputPath === undefined ||
cli.stdout
)
) {
process.stdout.write(output);
}
perfPrint('exit', ourbigbook_options)
if (!cli.watch) {
if (nonOurbigbookOptions.sequelize) await nonOurbigbookOptions.sequelize.close()
process.exit(nonOurbigbookOptions.had_error ? 1 : 0)
}
}
})().catch((e) => {
console.error(e);
process.exit(1);
})
}