New source found from dndbeyond.com
This commit is contained in:
+228
@@ -0,0 +1,228 @@
|
||||
/**
|
||||
* @import {Chunk, Event, Token} from 'micromark-util-types'
|
||||
*/
|
||||
|
||||
import { splice } from 'micromark-util-chunked';
|
||||
import { SpliceBuffer } from './lib/splice-buffer.js';
|
||||
|
||||
// Hidden API exposed for testing.
|
||||
export { SpliceBuffer } from './lib/splice-buffer.js';
|
||||
|
||||
/**
|
||||
* Tokenize subcontent.
|
||||
*
|
||||
* @param {Array<Event>} eventsArray
|
||||
* List of events.
|
||||
* @returns {boolean}
|
||||
* Whether subtokens were found.
|
||||
*/
|
||||
// eslint-disable-next-line complexity
|
||||
export function subtokenize(eventsArray) {
|
||||
/** @type {Record<string, number>} */
|
||||
const jumps = {};
|
||||
let index = -1;
|
||||
/** @type {Event} */
|
||||
let event;
|
||||
/** @type {number | undefined} */
|
||||
let lineIndex;
|
||||
/** @type {number} */
|
||||
let otherIndex;
|
||||
/** @type {Event} */
|
||||
let otherEvent;
|
||||
/** @type {Array<Event>} */
|
||||
let parameters;
|
||||
/** @type {Array<Event>} */
|
||||
let subevents;
|
||||
/** @type {boolean | undefined} */
|
||||
let more;
|
||||
const events = new SpliceBuffer(eventsArray);
|
||||
while (++index < events.length) {
|
||||
while (index in jumps) {
|
||||
index = jumps[index];
|
||||
}
|
||||
event = events.get(index);
|
||||
|
||||
// Add a hook for the GFM tasklist extension, which needs to know if text
|
||||
// is in the first content of a list item.
|
||||
if (index && event[1].type === "chunkFlow" && events.get(index - 1)[1].type === "listItemPrefix") {
|
||||
subevents = event[1]._tokenizer.events;
|
||||
otherIndex = 0;
|
||||
if (otherIndex < subevents.length && subevents[otherIndex][1].type === "lineEndingBlank") {
|
||||
otherIndex += 2;
|
||||
}
|
||||
if (otherIndex < subevents.length && subevents[otherIndex][1].type === "content") {
|
||||
while (++otherIndex < subevents.length) {
|
||||
if (subevents[otherIndex][1].type === "content") {
|
||||
break;
|
||||
}
|
||||
if (subevents[otherIndex][1].type === "chunkText") {
|
||||
subevents[otherIndex][1]._isInFirstContentOfListItem = true;
|
||||
otherIndex++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Enter.
|
||||
if (event[0] === 'enter') {
|
||||
if (event[1].contentType) {
|
||||
Object.assign(jumps, subcontent(events, index));
|
||||
index = jumps[index];
|
||||
more = true;
|
||||
}
|
||||
}
|
||||
// Exit.
|
||||
else if (event[1]._container) {
|
||||
otherIndex = index;
|
||||
lineIndex = undefined;
|
||||
while (otherIndex--) {
|
||||
otherEvent = events.get(otherIndex);
|
||||
if (otherEvent[1].type === "lineEnding" || otherEvent[1].type === "lineEndingBlank") {
|
||||
if (otherEvent[0] === 'enter') {
|
||||
if (lineIndex) {
|
||||
events.get(lineIndex)[1].type = "lineEndingBlank";
|
||||
}
|
||||
otherEvent[1].type = "lineEnding";
|
||||
lineIndex = otherIndex;
|
||||
}
|
||||
} else if (otherEvent[1].type === "linePrefix" || otherEvent[1].type === "listItemIndent") {
|
||||
// Move past.
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lineIndex) {
|
||||
// Fix position.
|
||||
event[1].end = {
|
||||
...events.get(lineIndex)[1].start
|
||||
};
|
||||
|
||||
// Switch container exit w/ line endings.
|
||||
parameters = events.slice(lineIndex, index);
|
||||
parameters.unshift(event);
|
||||
events.splice(lineIndex, index - lineIndex + 1, parameters);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The changes to the `events` buffer must be copied back into the eventsArray
|
||||
splice(eventsArray, 0, Number.POSITIVE_INFINITY, events.slice(0));
|
||||
return !more;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize embedded tokens.
|
||||
*
|
||||
* @param {SpliceBuffer<Event>} events
|
||||
* Events.
|
||||
* @param {number} eventIndex
|
||||
* Index.
|
||||
* @returns {Record<string, number>}
|
||||
* Gaps.
|
||||
*/
|
||||
function subcontent(events, eventIndex) {
|
||||
const token = events.get(eventIndex)[1];
|
||||
const context = events.get(eventIndex)[2];
|
||||
let startPosition = eventIndex - 1;
|
||||
/** @type {Array<number>} */
|
||||
const startPositions = [];
|
||||
let tokenizer = token._tokenizer;
|
||||
if (!tokenizer) {
|
||||
tokenizer = context.parser[token.contentType](token.start);
|
||||
if (token._contentTypeTextTrailing) {
|
||||
tokenizer._contentTypeTextTrailing = true;
|
||||
}
|
||||
}
|
||||
const childEvents = tokenizer.events;
|
||||
/** @type {Array<[number, number]>} */
|
||||
const jumps = [];
|
||||
/** @type {Record<string, number>} */
|
||||
const gaps = {};
|
||||
/** @type {Array<Chunk>} */
|
||||
let stream;
|
||||
/** @type {Token | undefined} */
|
||||
let previous;
|
||||
let index = -1;
|
||||
/** @type {Token | undefined} */
|
||||
let current = token;
|
||||
let adjust = 0;
|
||||
let start = 0;
|
||||
const breaks = [start];
|
||||
|
||||
// Loop forward through the linked tokens to pass them in order to the
|
||||
// subtokenizer.
|
||||
while (current) {
|
||||
// Find the position of the event for this token.
|
||||
while (events.get(++startPosition)[1] !== current) {
|
||||
// Empty.
|
||||
}
|
||||
startPositions.push(startPosition);
|
||||
if (!current._tokenizer) {
|
||||
stream = context.sliceStream(current);
|
||||
if (!current.next) {
|
||||
stream.push(null);
|
||||
}
|
||||
if (previous) {
|
||||
tokenizer.defineSkip(current.start);
|
||||
}
|
||||
if (current._isInFirstContentOfListItem) {
|
||||
tokenizer._gfmTasklistFirstContentOfListItem = true;
|
||||
}
|
||||
tokenizer.write(stream);
|
||||
if (current._isInFirstContentOfListItem) {
|
||||
tokenizer._gfmTasklistFirstContentOfListItem = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Unravel the next token.
|
||||
previous = current;
|
||||
current = current.next;
|
||||
}
|
||||
|
||||
// Now, loop back through all events (and linked tokens), to figure out which
|
||||
// parts belong where.
|
||||
current = token;
|
||||
while (++index < childEvents.length) {
|
||||
if (
|
||||
// Find a void token that includes a break.
|
||||
childEvents[index][0] === 'exit' && childEvents[index - 1][0] === 'enter' && childEvents[index][1].type === childEvents[index - 1][1].type && childEvents[index][1].start.line !== childEvents[index][1].end.line) {
|
||||
start = index + 1;
|
||||
breaks.push(start);
|
||||
// Help GC.
|
||||
current._tokenizer = undefined;
|
||||
current.previous = undefined;
|
||||
current = current.next;
|
||||
}
|
||||
}
|
||||
|
||||
// Help GC.
|
||||
tokenizer.events = [];
|
||||
|
||||
// If there’s one more token (which is the cases for lines that end in an
|
||||
// EOF), that’s perfect: the last point we found starts it.
|
||||
// If there isn’t then make sure any remaining content is added to it.
|
||||
if (current) {
|
||||
// Help GC.
|
||||
current._tokenizer = undefined;
|
||||
current.previous = undefined;
|
||||
} else {
|
||||
breaks.pop();
|
||||
}
|
||||
|
||||
// Now splice the events from the subtokenizer into the current events,
|
||||
// moving back to front so that splice indices aren’t affected.
|
||||
index = breaks.length;
|
||||
while (index--) {
|
||||
const slice = childEvents.slice(breaks[index], breaks[index + 1]);
|
||||
const start = startPositions.pop();
|
||||
jumps.push([start, start + slice.length - 1]);
|
||||
events.splice(start, 2, slice);
|
||||
}
|
||||
jumps.reverse();
|
||||
index = -1;
|
||||
while (++index < jumps.length) {
|
||||
gaps[adjust + jumps[index][0]] = adjust + jumps[index][1];
|
||||
adjust += jumps[index][1] - jumps[index][0] - 1;
|
||||
}
|
||||
return gaps;
|
||||
}
|
||||
+251
@@ -0,0 +1,251 @@
|
||||
/**
|
||||
* Some of the internal operations of micromark do lots of editing
|
||||
* operations on very large arrays. This runs into problems with two
|
||||
* properties of most circa-2020 JavaScript interpreters:
|
||||
*
|
||||
* - Array-length modifications at the high end of an array (push/pop) are
|
||||
* expected to be common and are implemented in (amortized) time
|
||||
* proportional to the number of elements added or removed, whereas
|
||||
* other operations (shift/unshift and splice) are much less efficient.
|
||||
* - Function arguments are passed on the stack, so adding tens of thousands
|
||||
* of elements to an array with `arr.push(...newElements)` will frequently
|
||||
* cause stack overflows. (see <https://stackoverflow.com/questions/22123769/rangeerror-maximum-call-stack-size-exceeded-why>)
|
||||
*
|
||||
* SpliceBuffers are an implementation of gap buffers, which are a
|
||||
* generalization of the "queue made of two stacks" idea. The splice buffer
|
||||
* maintains a cursor, and moving the cursor has cost proportional to the
|
||||
* distance the cursor moves, but inserting, deleting, or splicing in
|
||||
* new information at the cursor is as efficient as the push/pop operation.
|
||||
* This allows for an efficient sequence of splices (or pushes, pops, shifts,
|
||||
* or unshifts) as long such edits happen at the same part of the array or
|
||||
* generally sweep through the array from the beginning to the end.
|
||||
*
|
||||
* The interface for splice buffers also supports large numbers of inputs by
|
||||
* passing a single array argument rather passing multiple arguments on the
|
||||
* function call stack.
|
||||
*
|
||||
* @template T
|
||||
* Item type.
|
||||
*/
|
||||
export class SpliceBuffer {
|
||||
/**
|
||||
* @param {ReadonlyArray<T> | null | undefined} [initial]
|
||||
* Initial items (optional).
|
||||
* @returns
|
||||
* Splice buffer.
|
||||
*/
|
||||
constructor(initial) {
|
||||
/** @type {Array<T>} */
|
||||
this.left = initial ? [...initial] : [];
|
||||
/** @type {Array<T>} */
|
||||
this.right = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Array access;
|
||||
* does not move the cursor.
|
||||
*
|
||||
* @param {number} index
|
||||
* Index.
|
||||
* @return {T}
|
||||
* Item.
|
||||
*/
|
||||
get(index) {
|
||||
if (index < 0 || index >= this.left.length + this.right.length) {
|
||||
throw new RangeError('Cannot access index `' + index + '` in a splice buffer of size `' + (this.left.length + this.right.length) + '`');
|
||||
}
|
||||
if (index < this.left.length) return this.left[index];
|
||||
return this.right[this.right.length - index + this.left.length - 1];
|
||||
}
|
||||
|
||||
/**
|
||||
* The length of the splice buffer, one greater than the largest index in the
|
||||
* array.
|
||||
*/
|
||||
get length() {
|
||||
return this.left.length + this.right.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove and return `list[0]`;
|
||||
* moves the cursor to `0`.
|
||||
*
|
||||
* @returns {T | undefined}
|
||||
* Item, optional.
|
||||
*/
|
||||
shift() {
|
||||
this.setCursor(0);
|
||||
return this.right.pop();
|
||||
}
|
||||
|
||||
/**
|
||||
* Slice the buffer to get an array;
|
||||
* does not move the cursor.
|
||||
*
|
||||
* @param {number} start
|
||||
* Start.
|
||||
* @param {number | null | undefined} [end]
|
||||
* End (optional).
|
||||
* @returns {Array<T>}
|
||||
* Array of items.
|
||||
*/
|
||||
slice(start, end) {
|
||||
/** @type {number} */
|
||||
const stop = end === null || end === undefined ? Number.POSITIVE_INFINITY : end;
|
||||
if (stop < this.left.length) {
|
||||
return this.left.slice(start, stop);
|
||||
}
|
||||
if (start > this.left.length) {
|
||||
return this.right.slice(this.right.length - stop + this.left.length, this.right.length - start + this.left.length).reverse();
|
||||
}
|
||||
return this.left.slice(start).concat(this.right.slice(this.right.length - stop + this.left.length).reverse());
|
||||
}
|
||||
|
||||
/**
|
||||
* Mimics the behavior of Array.prototype.splice() except for the change of
|
||||
* interface necessary to avoid segfaults when patching in very large arrays.
|
||||
*
|
||||
* This operation moves cursor is moved to `start` and results in the cursor
|
||||
* placed after any inserted items.
|
||||
*
|
||||
* @param {number} start
|
||||
* Start;
|
||||
* zero-based index at which to start changing the array;
|
||||
* negative numbers count backwards from the end of the array and values
|
||||
* that are out-of bounds are clamped to the appropriate end of the array.
|
||||
* @param {number | null | undefined} [deleteCount=0]
|
||||
* Delete count (default: `0`);
|
||||
* maximum number of elements to delete, starting from start.
|
||||
* @param {Array<T> | null | undefined} [items=[]]
|
||||
* Items to include in place of the deleted items (default: `[]`).
|
||||
* @return {Array<T>}
|
||||
* Any removed items.
|
||||
*/
|
||||
splice(start, deleteCount, items) {
|
||||
/** @type {number} */
|
||||
const count = deleteCount || 0;
|
||||
this.setCursor(Math.trunc(start));
|
||||
const removed = this.right.splice(this.right.length - count, Number.POSITIVE_INFINITY);
|
||||
if (items) chunkedPush(this.left, items);
|
||||
return removed.reverse();
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove and return the highest-numbered item in the array, so
|
||||
* `list[list.length - 1]`;
|
||||
* Moves the cursor to `length`.
|
||||
*
|
||||
* @returns {T | undefined}
|
||||
* Item, optional.
|
||||
*/
|
||||
pop() {
|
||||
this.setCursor(Number.POSITIVE_INFINITY);
|
||||
return this.left.pop();
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts a single item to the high-numbered side of the array;
|
||||
* moves the cursor to `length`.
|
||||
*
|
||||
* @param {T} item
|
||||
* Item.
|
||||
* @returns {undefined}
|
||||
* Nothing.
|
||||
*/
|
||||
push(item) {
|
||||
this.setCursor(Number.POSITIVE_INFINITY);
|
||||
this.left.push(item);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts many items to the high-numbered side of the array.
|
||||
* Moves the cursor to `length`.
|
||||
*
|
||||
* @param {Array<T>} items
|
||||
* Items.
|
||||
* @returns {undefined}
|
||||
* Nothing.
|
||||
*/
|
||||
pushMany(items) {
|
||||
this.setCursor(Number.POSITIVE_INFINITY);
|
||||
chunkedPush(this.left, items);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts a single item to the low-numbered side of the array;
|
||||
* Moves the cursor to `0`.
|
||||
*
|
||||
* @param {T} item
|
||||
* Item.
|
||||
* @returns {undefined}
|
||||
* Nothing.
|
||||
*/
|
||||
unshift(item) {
|
||||
this.setCursor(0);
|
||||
this.right.push(item);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts many items to the low-numbered side of the array;
|
||||
* moves the cursor to `0`.
|
||||
*
|
||||
* @param {Array<T>} items
|
||||
* Items.
|
||||
* @returns {undefined}
|
||||
* Nothing.
|
||||
*/
|
||||
unshiftMany(items) {
|
||||
this.setCursor(0);
|
||||
chunkedPush(this.right, items.reverse());
|
||||
}
|
||||
|
||||
/**
|
||||
* Move the cursor to a specific position in the array. Requires
|
||||
* time proportional to the distance moved.
|
||||
*
|
||||
* If `n < 0`, the cursor will end up at the beginning.
|
||||
* If `n > length`, the cursor will end up at the end.
|
||||
*
|
||||
* @param {number} n
|
||||
* Position.
|
||||
* @return {undefined}
|
||||
* Nothing.
|
||||
*/
|
||||
setCursor(n) {
|
||||
if (n === this.left.length || n > this.left.length && this.right.length === 0 || n < 0 && this.left.length === 0) return;
|
||||
if (n < this.left.length) {
|
||||
// Move cursor to the this.left
|
||||
const removed = this.left.splice(n, Number.POSITIVE_INFINITY);
|
||||
chunkedPush(this.right, removed.reverse());
|
||||
} else {
|
||||
// Move cursor to the this.right
|
||||
const removed = this.right.splice(this.left.length + this.right.length - n, Number.POSITIVE_INFINITY);
|
||||
chunkedPush(this.left, removed.reverse());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Avoid stack overflow by pushing items onto the stack in segments
|
||||
*
|
||||
* @template T
|
||||
* Item type.
|
||||
* @param {Array<T>} list
|
||||
* List to inject into.
|
||||
* @param {ReadonlyArray<T>} right
|
||||
* Items to inject.
|
||||
* @return {undefined}
|
||||
* Nothing.
|
||||
*/
|
||||
function chunkedPush(list, right) {
|
||||
/** @type {number} */
|
||||
let chunkStart = 0;
|
||||
if (right.length < 10000) {
|
||||
list.push(...right);
|
||||
} else {
|
||||
while (chunkStart < right.length) {
|
||||
list.push(...right.slice(chunkStart, chunkStart + 10000));
|
||||
chunkStart += 10000;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user