Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • TransFem-org/sfm-js
  • dakkar/sfm-js
  • septicake/sfm-js-docs
3 results
Show changes
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 2570 additions and 780 deletions
src/@types/twemoji.d.ts 0 → 100644
View file @ f5d7cb10
declare module '@twemoji/parser/dist/lib/regex' {
const regex: RegExp;
export default regex;
}
src/api.ts
View file @ f5d7cb10
import peg from 'peggy';
import { MfmNode, MfmPlainNode } from './node';
import { stringifyNode, stringifyTree, inspectOne } from './util';
const parser: peg.Parser = require('./parser');
import { fullParser, simpleParser } from './internal';
import { inspectOne, stringifyNode, stringifyTree } from './internal/util';
import { MfmNode, MfmSimpleNode } from './node';
/**
* Generates a MfmNode tree from the MFM string.
*/
export function parse(input: string): MfmNode[] {
const nodes = parser.parse(input, { startRule: 'fullParser' });
export function parse(input: string, opts: Partial<{ nestLimit: number; }> = {}): MfmNode[] {
const nodes = fullParser(input, {
nestLimit: opts.nestLimit,
});
return nodes;
}
/**
* Generates a MfmNode tree of plain from the MFM string.
* Generates a MfmSimpleNode tree from the MFM string.
*
* "Simple" MFM only recognises text and emojis
*/
export function parsePlain(input: string): MfmPlainNode[] {
const nodes = parser.parse(input, { startRule: 'plainParser' });
export function parseSimple(input: string): MfmSimpleNode[] {
const nodes = simpleParser(input);
return nodes;
}
/**
* Generates a MFM string from the MfmNode tree.
*
* Notice that the result of `toString(parse(someString))` will very
* probably not be identical to `someString`
*/
export function toString(tree: MfmNode[]): string
export function toString(node: MfmNode): string
......@@ -36,6 +41,17 @@ export function toString(node: MfmNode | MfmNode[]): string {
/**
* Inspects the MfmNode tree.
*
* This is the visitor pattern. Your `action` will be called on each
* node of the tree, as a depth-first pre-visit:
*
* ```
* A
* +B
* |+C
* |+D
* +E
* ```
*/
export function inspect(node: MfmNode, action: (node: MfmNode) => void): void
export function inspect(nodes: MfmNode[], action: (node: MfmNode) => void): void
......
src/cli/parse.ts
View file @ f5d7cb10
......@@ -2,7 +2,7 @@ import { performance } from 'perf_hooks';
import inputLine, { InputCanceledError } from './misc/inputLine';
import { parse } from '..';
async function entryPoint() {
async function entryPoint(): Promise<void> {
console.log('intaractive parser');
while (true) {
......@@ -39,8 +39,9 @@ async function entryPoint() {
console.log();
}
}
entryPoint()
.catch(err => {
console.log(err);
process.exit(1);
});
.catch(err => {
console.log(err);
process.exit(1);
});
src/cli/parsePlain.ts → src/cli/parseSimple.ts
View file @ f5d7cb10
import { performance } from 'perf_hooks';
import inputLine, { InputCanceledError } from './misc/inputLine';
import { parsePlain } from '..';
import { parseSimple } from '..';
async function entryPoint() {
console.log('intaractive plain parser');
async function entryPoint(): Promise<void> {
console.log('intaractive simple parser');
while (true) {
let input: string;
......@@ -26,7 +26,7 @@ async function entryPoint() {
try {
const parseTimeStart = performance.now();
const result = parsePlain(input);
const result = parseSimple(input);
const parseTimeEnd = performance.now();
console.log(JSON.stringify(result));
const parseTime = (parseTimeEnd - parseTimeStart).toFixed(3);
......@@ -39,8 +39,9 @@ async function entryPoint() {
console.log();
}
}
entryPoint()
.catch(err => {
console.log(err);
process.exit(1);
});
.catch(err => {
console.log(err);
process.exit(1);
});
src/index.ts
View file @ f5d7cb10
export {
parse,
parsePlain,
parseSimple,
toString,
inspect,
extract
extract,
} from './api';
export {
NodeType,
MfmNode,
MfmSimpleNode,
MfmBlock,
MfmInline
MfmInline,
} from './node';
export {
......@@ -35,7 +36,8 @@ export {
MfmUrl,
MfmLink,
MfmFn,
MfmText
MfmPlain,
MfmText,
} from './node';
export {
......@@ -60,5 +62,6 @@ export {
N_URL,
LINK,
FN,
TEXT
PLAIN,
TEXT,
} from './node';
src/internal/core/index.ts 0 → 100644
View file @ f5d7cb10
//
// Parsimmon-like stateful parser combinators
//
/**
* Holds the information from a successful parse: a parsed node, and the position where the parsing stopped.
*/
export type Success<T> = {
success: true;
value: T;
index: number;
};
/**
* Represents a failed parse.
*/
export type Failure = { success: false };
/**
* Possible results from a parse.
*/
export type Result<T> = Success<T> | Failure;
/**
* Parser state: should we print what we're doing? are we inside a link label? how deep can we go? how deep are we?
*/
interface State {
trace?: boolean,
linkLabel?: boolean,
nestLimit: number,
depth: number,
}
/**
* The function that actually does the parsing (of the given string, from the given position)
*/
export type ParserHandler<T> = (input: string, index: number, state: State) => Result<T>
/**
* Simplified constructor for `Success`
*
* @param index The index of the success.
* @param value The value of the success.
* @returns A {@link Success} object.
*/
export function success<T>(index: number, value: T): Success<T> {
return {
success: true,
value: value,
index: index,
};
}
/**
* Simplified constructor for `Failure`
*
* @returns A {@link Failure} object.
*/
export function failure(): Failure {
return { success: false };
}
/**
* The parser class. Delegates most of the parsing to the `handler`, but provides combinators on top of it.
*/
export class Parser<T> {
public name?: string;
public handler: ParserHandler<T>;
constructor(handler: ParserHandler<T>, name?: string) {
this.handler = (input, index, state) : Failure | Success<T> => {
if (state.trace && this.name != null) {
const pos = `${index}`;
console.log(`${pos.padEnd(6, ' ')}enter ${this.name}`);
const result = handler(input, index, state);
if (result.success) {
const pos = `${index}:${result.index}`;
console.log(`${pos.padEnd(6, ' ')}match ${this.name}`);
} else {
const pos = `${index}`;
console.log(`${pos.padEnd(6, ' ')}fail ${this.name}`);
}
return result;
}
return handler(input, index, state);
};
this.name = name;
}
/**
* Returns a new parser, just like `this` parser, but the values of
* successful parses are passed through the given function.
*
* @param fn The function used to map the output of the parser.
* @returns The result of the parser mapped with `fn`.
*/
map<U>(fn: (value: T) => U): Parser<U> {
return new Parser((input, index, state) => {
const result = this.handler(input, index, state);
if (!result.success) {
return result;
}
return success(result.index, fn(result.value));
});
}
/**
* Returns a new parser, just like `this` parser, but the result is just the matched input text.
*
* @returns The plaintext related to the successful parse, and a {@link Failure} if the parse failed.
*/
text(): Parser<string> {
return new Parser((input, index, state) => {
const result = this.handler(input, index, state);
if (!result.success) {
return result;
}
const text = input.slice(index, result.index);
return success(result.index, text);
});
}
/**
* Returns a new parser, that matches at least `min` repetitions of `this` parser.
*
* @param min The minimum amount of times this parse must succeed to return a {@link Success}.
* @returns A Parser that returns a {@link Success} object if it matches enough times, and a {@link Failure} otherwise.
*/
many(min: number): Parser<T[]> {
return new Parser((input, index, state) => {
let result;
let latestIndex = index;
const accum: T[] = [];
while (latestIndex < input.length) {
result = this.handler(input, latestIndex, state);
if (!result.success) {
break;
}
latestIndex = result.index;
accum.push(result.value);
}
if (accum.length < min) {
return failure();
}
return success(latestIndex, accum);
});
}
/**
* Returns a new parser, matches at least `min` repetitions of `this` parser, separated by `separator`.
*
* @param separator The parser representing the separator that must appear between this parser's value.
* @param min The minimum amount of times this parse must succeed to return a {@link Success}.
* @returns A Parser that returns a {@link Success} object if it matches enough times, and a {@link Failure} otherwise.
*/
sep(separator: Parser<unknown>, min: number): Parser<T[]> {
if (min < 1) {
throw new Error('"min" must be a value greater than or equal to 1.');
}
return seq(
this,
seq(
separator,
this,
).select(1).many(min - 1),
).map(result => [result[0], ...result[1]]);
}
/**
* Returns a new parser, whose result is the part of `this` parser's result selected by the given key.
* (so if `this` produces a success value like `{foo:1}`, `this.select('foo')` would result in `1`)
*
* @param key The value used to select a part of the result
* @returns The result of the parser subscripted by `key`
*/
select<K extends keyof T>(key: K): Parser<T[K]> {
return this.map(v => v[key]);
}
/**
* Returns a new parser, just like `this` parser, but returns a null success on failure.
*
* @returns A Parser that always returns a {@link Success} object, maybe with a `null` inside.
*/
option(): Parser<T | null> {
return alt([
this,
succeeded(null),
]);
}
}
/**
* Construct a {@link Parser} that matches the supplied string.
*
* @param value The string that the returned {@link Parser} checks for.
* @returns A {@link Parser} that matches the supplied string.
*/
export function str<T extends string>(value: T): Parser<T> {
return new Parser((input, index, _state) => {
if ((input.length - index) < value.length) {
return failure();
}
if (input.substr(index, value.length) !== value) {
return failure();
}
return success(index + value.length, value);
});
}
/**
* Construct a {@link Parser} that matches the supplied regular expression.
*
* @param pattern The regular expression that the returned {@link Parser} tries to match.
* @returns A {@link Parser} that checks if the input matches the supplied regular expression.
*/
export function regexp<T extends RegExp>(pattern: T): Parser<string> {
const re = RegExp(`^(?:${pattern.source})`, pattern.flags);
return new Parser((input, index, _state) => {
const text = input.slice(index);
const result = re.exec(text);
if (result == null) {
return failure();
}
return success(index + result[0].length, result[0]);
});
}
type ParsedType<T extends Parser<unknown>> = T extends Parser<infer U> ? U : never;
export type SeqParseResult<T extends unknown[]> =
T extends [] ? []
: T extends [infer F, ...infer R]
? (
F extends Parser<unknown> ? [ParsedType<F>, ...SeqParseResult<R>] : [unknown, ...SeqParseResult<R>]
)
: unknown[];
/**
* Construct a {@link Parser} that goes through the parsers provided, in order, and checks that they all
* succeed. A {@link Failure} object is returned if any of the parsers fails.
*
* @param parsers The array of {@link Parser Parsers} that are checked to see if it succeeds.
* @returns A {@link Parser} that runs through the parsers in the order that they were provided and returns
* all the values they returned.
*/
export function seq<Parsers extends Parser<unknown>[]>(...parsers: Parsers): Parser<SeqParseResult<Parsers>> {
return new Parser((input, index, state) => {
let result;
let latestIndex = index;
const accum = [];
for (let i = 0; i < parsers.length; i++) {
result = parsers[i].handler(input, latestIndex, state);
if (!result.success) {
return result;
}
latestIndex = result.index;
accum.push(result.value);
}
return success(latestIndex, accum as SeqParseResult<Parsers>);
});
}
/**
* Construct a {@link Parser} that goes through the parsers provided, in order, and checks if any succeed.
* The returned parser produces the result of the first element of `parsers` to succeed, or a failure if none do.
*
* @param parsers The {@link Parser Parsers} that should be used.
* @returns A {@link Parser} that returns the first {@link Success} from the supplied parsers.
*/
export function alt<Parsers extends Parser<unknown>[]>(parsers: Parsers): Parser<ParsedType<Parsers[number]>> {
return new Parser<ParsedType<Parsers[number]>>((input, index, state): Result<ParsedType<Parsers[number]>> => {
for (let i = 0; i < parsers.length; i++) {
const parser: Parsers[number] = parsers[i];
const result = parser.handler(input, index, state);
if (result.success) {
return result as Result<ParsedType<Parsers[number]>>;
}
}
return failure();
});
}
/**
* Construct a constant {@link Parser} that always succeeds with the given value.
*
* @param value The value to be used in the returned {@link Success} object.
* @returns A {@link Parser} that always returns a {@link Success} with the specified value.
*/
function succeeded<T>(value: T): Parser<T> {
return new Parser((_input, index, _state) => {
return success(index, value);
});
}
/**
* Construct a {@link Parser} that succeeds when the given parser fails, and vice versa.
*
* @param parser The {@link Parser} to be matched.
* @returns A {@link Success} with the value `null` if the parser fails, or a {@link Failure} if it succeeds.
*/
export function notMatch(parser: Parser<unknown>): Parser<null> {
return new Parser((input, index, state) => {
const result = parser.handler(input, index, state);
return !result.success
? success(index, null)
: failure();
});
}
/**
* Construct a {@link Parser} just like `parserIncluded`, but fails if
* `parserExcluded` succeeds. So it matches the "included" language,
* minus the "excluded" language.
*
* @param parserIncluded The {@link Parser} that should succeed
* @param parserExcluded The {@link Parser} that should fail
* @returns A {@link Failure} object if `parserExcluded` succeeds, or if `parserIncluded` fails, and a {@link Success} object
* otherwise.
*/
export function difference(parserIncluded: Parser<string>, parserExcluded: Parser<string>): Parser<string> {
return new Parser((input, index, state) => {
const exclude = parserExcluded.handler(input, index, state);
if (exclude.success) {
return failure();
}
return parserIncluded.handler(input, index, state);
});
}
/** A {@link Parser} that matches the carriage return character `\r`. */
export const cr = str('\r');
/** A {@link Parser} that matches the line feed character `\n`. */
export const lf = str('\n');
/** A {@link Parser} that matches the character sequence `\r\n`. */
export const crlf = str('\r\n');
/** A {@link Parser} that matches any valid new line sequence. */
export const newline = alt([crlf, cr, lf]);
/**
* A {@link Parser} that matches a character.
*/
export const char = new Parser((input, index, _state) => {
if ((input.length - index) < 1) {
return failure();
}
const value = input.charAt(index);
return success(index + 1, value);
});
/**
* A {@link Parser} that checks that we are at the beginning of a line or of the input.
*/
export const lineBegin = new Parser((input, index, state) => {
if (index === 0) {
return success(index, null);
}
if (cr.handler(input, index - 1, state).success) {
return success(index, null);
}
if (lf.handler(input, index - 1, state).success) {
return success(index, null);
}
return failure();
});
/**
* A {@link Parser} that checks if we are at the end of a line or of the input.
*/
export const lineEnd = new Parser((input, index, state) => {
if (index === input.length) {
return success(index, null);
}
if (cr.handler(input, index, state).success) {
return success(index, null);
}
if (lf.handler(input, index, state).success) {
return success(index, null);
}
return failure();
});
/**
* Lazily define a parser. This allows for self-recursive parsers (see for example the `url` and `hashtag` rules)
*
* @param fn A function that returns the actual {@link Parser}
* @returns A {@link Parser} that becomes the actual parser on its first use
*/
export function lazy<T>(fn: () => Parser<T>): Parser<T> {
// Convert a parser generator into a parser: when `parser` is first
// invoked, it replaces its own handler with the real one, and calls
// it. On all subsequent invocations, the real handler will be
// called directly
const parser: Parser<T> = new Parser((input, index, state) => {
parser.handler = fn().handler;
return parser.handler(input, index, state);
});
return parser;
}
//type Syntax<T> = (rules: Record<string, Parser<T>>) => Parser<T>;
//type SyntaxReturn<T> = T extends (rules: Record<string, Parser<any>>) => infer R ? R : never;
//export function createLanguage2<T extends Record<string, Syntax<any>>>(syntaxes: T): { [K in keyof T]: SyntaxReturn<T[K]> } {
type ParserTable<T> = { [K in keyof T]: Parser<T[K]> };
// TODO: 関数の型宣言をいい感じにしたい
export function createLanguage<T>(syntaxes: { [K in keyof T]: (r: ParserTable<T>) => Parser<T[K]> }): ParserTable<T> {
// @ts-expect-error initializing object so type error here
const rules: ParserTable<T> = {};
for (const key of Object.keys(syntaxes) as (keyof T & string)[]) {
rules[key] = lazy(() => {
const parser = syntaxes[key](rules);
if (parser == null) {
throw new Error('syntax must return a parser.');
}
parser.name = key;
return parser;
});
}
return rules;
}
src/internal/index.ts 0 → 100644
View file @ f5d7cb10
import * as M from '..';
import { language } from './parser';
import { mergeText } from './util';
/**
* A type representing the available options for the full parser.
*/
export type FullParserOpts = {
nestLimit?: number;
};
/**
* A function that parses through the input text with the full parser and returns the AST representing the
* result.
*
* @param input The input string to parse.
* @param opts The options used for the parsing.
* @returns An array of nodes representing the resulting styles.
*/
export function fullParser(input: string, opts: FullParserOpts): M.MfmNode[] {
const result = language.fullParser.handler(input, 0, {
nestLimit: (opts.nestLimit != null) ? opts.nestLimit : 20,
depth: 0,
linkLabel: false,
trace: false,
});
if (!result.success) throw new Error('Unexpected parse error');
return mergeText(result.value);
}
/**
* A function that parses through the input text with the simple parser and returns the AST representing the
* result.
*
* @param input The input string to parse.
* @returns An array of simple nodes represennting the resulting styles.
*/
export function simpleParser(input: string): M.MfmSimpleNode[] {
const result = language.simpleParser.handler(input, 0, {
depth: 0,
nestLimit: 1 / 0, // reliable infinite
});
if (!result.success) throw new Error('Unexpected parse error');
return mergeText(result.value);
}
src/internal/parser.ts 0 → 100644
View file @ f5d7cb10
This diff is collapsed.
src/util.ts → src/internal/util.ts
View file @ f5d7cb10
import { isMfmBlock, MfmNode, TEXT } from './node';
import { isMfmBlock, MfmInline, MfmNode, MfmText, TEXT } from '../node';
export function mergeText(nodes: (MfmNode | string)[]): MfmNode[] {
const dest: MfmNode[] = [];
type ArrayRecursive<T> = T | Array<ArrayRecursive<T>>;
export function mergeText<T extends MfmNode>(nodes: ArrayRecursive<((T extends MfmInline ? MfmInline : MfmNode) | string)>[]): (T | MfmText)[] {
const dest: (T | MfmText)[] = [];
const storedChars: string[] = [];
/**
* Generate a text node from the stored chars, And push it.
*/
function generateText() {
function generateText(): void {
if (storedChars.length > 0) {
dest.push(TEXT(storedChars.join('')));
storedChars.length = 0;
}
}
for (const node of nodes) {
if (typeof node == 'string') {
const flatten = nodes.flat(1) as (string | T)[];
for (const node of flatten) {
if (typeof node === 'string') {
// Store the char.
storedChars.push(node);
}
else if (!Array.isArray(node) && node.type === 'text') {
storedChars.push((node as MfmText).props.text);
}
else {
generateText();
dest.push(node);
......@@ -30,7 +36,7 @@ export function mergeText(nodes: (MfmNode | string)[]): MfmNode[] {
}
export function stringifyNode(node: MfmNode): string {
switch(node.type) {
switch (node.type) {
// block
case 'quote': {
return stringifyTree(node.children).split('\n').map(line => `> ${line}`).join('\n');
......@@ -79,7 +85,12 @@ export function stringifyNode(node: MfmNode): string {
return `#${ node.props.hashtag }`;
}
case 'url': {
return node.props.url;
if (node.props.brackets) {
return `<${ node.props.url }>`;
}
else {
return node.props.url;
}
}
case 'link': {
const prefix = node.props.silent ? '?' : '';
......@@ -98,6 +109,9 @@ export function stringifyNode(node: MfmNode): string {
const args = (argFields.length > 0) ? '.' + argFields.join(',') : '';
return `$[${ node.props.name }${ args } ${ stringifyTree(node.children) }]`;
}
case 'plain': {
return `<plain>\n${ stringifyTree(node.children) }\n</plain>`;
}
case 'text': {
return node.props.text;
}
......@@ -109,10 +123,10 @@ enum stringifyState {
none = 0,
inline,
block
};
}
export function stringifyTree(nodes: MfmNode[]): string {
let dest: MfmNode[] = [];
const dest: MfmNode[] = [];
let state: stringifyState = stringifyState.none;
for (const node of nodes) {
......@@ -124,15 +138,15 @@ export function stringifyTree(nodes: MfmNode[]): string {
// block -> inline : Yes
// block -> block : Yes
let pushLf: boolean = true;
let pushLf = true;
if (isMfmBlock(node)) {
if (state == stringifyState.none) {
if (state === stringifyState.none) {
pushLf = false;
}
state = stringifyState.block;
}
else {
if (state == stringifyState.none || state == stringifyState.inline) {
if (state === stringifyState.none || state === stringifyState.inline) {
pushLf = false;
}
state = stringifyState.inline;
......@@ -147,7 +161,7 @@ export function stringifyTree(nodes: MfmNode[]): string {
return dest.map(n => stringifyNode(n)).join('');
}
export function inspectOne(node: MfmNode, action: (node: MfmNode) => void) {
export function inspectOne(node: MfmNode, action: (node: MfmNode) => void): void {
action(node);
if (node.children != null) {
for (const child of node.children) {
......@@ -155,39 +169,3 @@ export function inspectOne(node: MfmNode, action: (node: MfmNode) => void) {
}
}
}
//
// dynamic consuming
//
/*
1. If you want to consume 3 chars, call the setConsumeCount.
```
setConsumeCount(3);
```
2. And the rule to consume the input is as below:
```
rule = (&{ return consumeDynamically(); } .)+
```
*/
let consumeCount = 0;
/**
* set the length of dynamic consuming.
*/
export function setConsumeCount(count: number) {
consumeCount = count;
}
/**
* consume the input and returns matching result.
*/
export function consumeDynamically() {
const matched = (consumeCount > 0);
if (matched) {
consumeCount--;
}
return matched;
}
src/node.ts
View file @ f5d7cb10
export type MfmNode = MfmBlock | MfmInline;
export type MfmPlainNode = MfmUnicodeEmoji | MfmEmojiCode | MfmText;
export type MfmSimpleNode = MfmUnicodeEmoji | MfmEmojiCode | MfmText | MfmPlain;
export type MfmBlock = MfmQuote | MfmSearch | MfmCodeBlock | MfmMathBlock | MfmCenter;
......@@ -11,10 +11,10 @@ export function isMfmBlock(node: MfmNode): node is MfmBlock {
export type MfmQuote = {
type: 'quote';
props?: { };
props?: Record<string, unknown>;
children: MfmNode[];
};
export const QUOTE = (children: MfmNode[]): NodeType<'quote'> => { return { type:'quote', children }; };
export const QUOTE = (children: MfmNode[]): NodeType<'quote'> => { return { type: 'quote', children }; };
export type MfmSearch = {
type: 'search';
......@@ -24,7 +24,7 @@ export type MfmSearch = {
};
children?: [];
};
export const SEARCH = (query: string, content: string): NodeType<'search'> => { return { type:'search', props: { query, content } }; };
export const SEARCH = (query: string, content: string): NodeType<'search'> => { return { type: 'search', props: { query, content } }; };
export type MfmCodeBlock = {
type: 'blockCode';
......@@ -34,7 +34,7 @@ export type MfmCodeBlock = {
};
children?: [];
};
export const CODE_BLOCK = (code: string, lang: string | null): NodeType<'blockCode'> => { return { type:'blockCode', props: { code, lang } }; };
export const CODE_BLOCK = (code: string, lang: string | null): NodeType<'blockCode'> => { return { type: 'blockCode', props: { code, lang } }; };
export type MfmMathBlock = {
type: 'mathBlock';
......@@ -43,17 +43,17 @@ export type MfmMathBlock = {
};
children?: [];
};
export const MATH_BLOCK = (formula: string): NodeType<'mathBlock'> => { return { type:'mathBlock', props: { formula } }; };
export const MATH_BLOCK = (formula: string): NodeType<'mathBlock'> => { return { type: 'mathBlock', props: { formula } }; };
export type MfmCenter = {
type: 'center';
props?: { };
props?: Record<string, unknown>;
children: MfmInline[];
};
export const CENTER = (children: MfmInline[]): NodeType<'center'> => { return { type:'center', children }; };
export const CENTER = (children: MfmInline[]): NodeType<'center'> => { return { type: 'center', children }; };
export type MfmInline = MfmUnicodeEmoji | MfmEmojiCode | MfmBold | MfmSmall | MfmItalic | MfmStrike |
MfmInlineCode | MfmMathInline | MfmMention | MfmHashtag | MfmUrl | MfmLink | MfmFn | MfmText;
MfmInlineCode | MfmMathInline | MfmMention | MfmHashtag | MfmUrl | MfmLink | MfmFn | MfmPlain | MfmText;
export type MfmUnicodeEmoji = {
type: 'unicodeEmoji';
......@@ -62,7 +62,7 @@ export type MfmUnicodeEmoji = {
};
children?: [];
};
export const UNI_EMOJI = (value: string): NodeType<'unicodeEmoji'> => { return { type:'unicodeEmoji', props: { emoji: value } }; };
export const UNI_EMOJI = (value: string): NodeType<'unicodeEmoji'> => { return { type: 'unicodeEmoji', props: { emoji: value } }; };
export type MfmEmojiCode = {
type: 'emojiCode';
......@@ -71,35 +71,35 @@ export type MfmEmojiCode = {
};
children?: [];
};
export const EMOJI_CODE = (name: string): NodeType<'emojiCode'> => { return { type:'emojiCode', props: { name: name } }; };
export const EMOJI_CODE = (name: string): NodeType<'emojiCode'> => { return { type: 'emojiCode', props: { name: name } }; };
export type MfmBold = {
type: 'bold';
props?: { };
props?: Record<string, unknown>;
children: MfmInline[];
};
export const BOLD = (children: MfmInline[]): NodeType<'bold'> => { return { type:'bold', children }; };
export const BOLD = (children: MfmInline[]): NodeType<'bold'> => { return { type: 'bold', children }; };
export type MfmSmall = {
type: 'small';
props?: { };
props?: Record<string, unknown>;
children: MfmInline[];
};
export const SMALL = (children: MfmInline[]): NodeType<'small'> => { return { type:'small', children }; };
export const SMALL = (children: MfmInline[]): NodeType<'small'> => { return { type: 'small', children }; };
export type MfmItalic = {
type: 'italic';
props?: { };
props?: Record<string, unknown>;
children: MfmInline[];
};
export const ITALIC = (children: MfmInline[]): NodeType<'italic'> => { return { type:'italic', children }; };
export const ITALIC = (children: MfmInline[]): NodeType<'italic'> => { return { type: 'italic', children }; };
export type MfmStrike = {
type: 'strike';
props?: { };
props?: Record<string, unknown>;
children: MfmInline[];
};
export const STRIKE = (children: MfmInline[]): NodeType<'strike'> => { return { type:'strike', children }; };
export const STRIKE = (children: MfmInline[]): NodeType<'strike'> => { return { type: 'strike', children }; };
export type MfmInlineCode = {
type: 'inlineCode';
......@@ -108,7 +108,7 @@ export type MfmInlineCode = {
};
children?: [];
};
export const INLINE_CODE = (code: string): NodeType<'inlineCode'> => { return { type:'inlineCode', props: { code } }; };
export const INLINE_CODE = (code: string): NodeType<'inlineCode'> => { return { type: 'inlineCode', props: { code } }; };
export type MfmMathInline = {
type: 'mathInline';
......@@ -117,7 +117,7 @@ export type MfmMathInline = {
};
children?: [];
};
export const MATH_INLINE = (formula: string): NodeType<'mathInline'> => { return { type:'mathInline', props: { formula } }; };
export const MATH_INLINE = (formula: string): NodeType<'mathInline'> => { return { type: 'mathInline', props: { formula } }; };
export type MfmMention = {
type: 'mention';
......@@ -128,7 +128,7 @@ export type MfmMention = {
};
children?: [];
};
export const MENTION = (username: string, host: string | null, acct: string): NodeType<'mention'> => { return { type:'mention', props: { username, host, acct } }; };
export const MENTION = (username: string, host: string | null, acct: string): NodeType<'mention'> => { return { type: 'mention', props: { username, host, acct } }; };
export type MfmHashtag = {
type: 'hashtag';
......@@ -137,16 +137,21 @@ export type MfmHashtag = {
};
children?: [];
};
export const HASHTAG = (value: string): NodeType<'hashtag'> => { return { type:'hashtag', props: { hashtag: value } }; };
export const HASHTAG = (value: string): NodeType<'hashtag'> => { return { type: 'hashtag', props: { hashtag: value } }; };
export type MfmUrl = {
type: 'url';
props: {
url: string;
brackets?: boolean;
};
children?: [];
};
export const N_URL = (value: string): NodeType<'url'> => { return { type:'url', props: { url: value } }; };
export const N_URL = (value: string, brackets?: boolean): NodeType<'url'> => {
const node: MfmUrl = { type: 'url', props: { url: value } };
if (brackets) node.props.brackets = brackets;
return node;
};
export type MfmLink = {
type: 'link';
......@@ -156,7 +161,7 @@ export type MfmLink = {
};
children: MfmInline[];
};
export const LINK = (silent: boolean, url: string, children: MfmInline[]): NodeType<'link'> => { return { type:'link', props: { silent, url }, children }; };
export const LINK = (silent: boolean, url: string, children: MfmInline[]): NodeType<'link'> => { return { type: 'link', props: { silent, url }, children }; };
export type MfmFn = {
type: 'fn';
......@@ -166,7 +171,14 @@ export type MfmFn = {
};
children: MfmInline[];
};
export const FN = (name: string, args: MfmFn['props']['args'], children: MfmFn['children']): NodeType<'fn'> => { return { type:'fn', props: { name, args }, children }; };
export const FN = (name: string, args: MfmFn['props']['args'], children: MfmFn['children']): NodeType<'fn'> => { return { type: 'fn', props: { name, args }, children }; };
export type MfmPlain = {
type: 'plain';
props?: Record<string, unknown>;
children: MfmText[];
};
export const PLAIN = (text: string): NodeType<'plain'> => { return { type: 'plain', children: [TEXT(text)] }; };
export type MfmText = {
type: 'text';
......@@ -175,7 +187,7 @@ export type MfmText = {
};
children?: [];
};
export const TEXT = (value: string): NodeType<'text'> => { return { type:'text', props: { text: value } }; };
export const TEXT = (value: string): NodeType<'text'> => { return { type: 'text', props: { text: value } }; };
export type NodeType<T extends MfmNode['type']> =
T extends 'quote' ? MfmQuote :
......@@ -196,5 +208,6 @@ export type NodeType<T extends MfmNode['type']> =
T extends 'url' ? MfmUrl :
T extends 'link' ? MfmLink :
T extends 'fn' ? MfmFn :
T extends 'plain' ? MfmPlain :
T extends 'text' ? MfmText :
never;
src/parser.pegjs deleted 100644 → 0
View file @ 07f6d94e
{
const {
// block
QUOTE,
SEARCH,
CODE_BLOCK,
MATH_BLOCK,
CENTER,
// inline
UNI_EMOJI,
EMOJI_CODE,
BOLD,
SMALL,
ITALIC,
STRIKE,
INLINE_CODE,
MATH_INLINE,
MENTION,
HASHTAG,
N_URL,
LINK,
FN,
TEXT
} = require('./node');
const {
mergeText,
setConsumeCount,
consumeDynamically
} = require('./util');
function applyParser(input, startRule) {
let parseFunc = peg$parse;
return parseFunc(input, startRule ? { startRule } : { });
}
// emoji
const emojiRegex = require('twemoji-parser/dist/lib/regex').default;
const anchoredEmojiRegex = RegExp(`^(?:${emojiRegex.source})`);
/**
* check if the input matches the emoji regexp.
* if they match, set the byte length of the emoji.
*/
function matchUnicodeEmoji() {
const offset = location().start.offset;
const src = input.substr(offset);
const result = anchoredEmojiRegex.exec(src);
if (result != null) {
setConsumeCount(result[0].length); // length(utf-16 byte length) of emoji sequence.
return true;
}
return false;
}
}
//
// parsers
//
fullParser
= nodes:(&. n:full { return n; })* { return mergeText(nodes); }
plainParser
= nodes:(&. n:plain { return n; })* { return mergeText(nodes); }
inlineParser
= nodes:(&. n:inline { return n; })* { return mergeText(nodes); }
//
// syntax list
//
full
= quote // block
/ codeBlock // block
/ mathBlock // block
/ center // block
/ emojiCode
/ unicodeEmoji
/ big
/ bold
/ small
/ italic
/ strike
/ inlineCode
/ mathInline
/ mention
/ hashtag
/ url
/ fnVer2
/ link
/ fnVer1
/ search // block
/ inlineText
inline
= emojiCode
/ unicodeEmoji
/ big
/ bold
/ small
/ italic
/ strike
/ inlineCode
/ mathInline
/ mention
/ hashtag
/ url
/ fnVer2
/ link
/ fnVer1
/ inlineText
plain
= emojiCode
/ unicodeEmoji
/ plainText
//
// block rules
//
// block: quote
quote
= &(BEGIN ">") q:quoteInner { return q; }
quoteInner
= head:quoteMultiLine tails:quoteMultiLine+
{
const children = applyParser([head, ...tails].join('\n'), 'fullParser');
return QUOTE(children);
}
/ line:quoteLine
{
const children = applyParser(line, 'fullParser');
return QUOTE(children);
}
quoteMultiLine
= quoteLine / quoteEmptyLine
quoteLine
= BEGIN ">" _? text:$(CHAR+) END { return text; }
quoteEmptyLine
= BEGIN ">" _? END { return ''; }
// block: search
search
= BEGIN q:searchQuery sp:_ key:searchKey END
{
return SEARCH(q, `${ q }${ sp }${ key }`);
}
searchQuery
= (!(_ searchKey END) CHAR)+ { return text(); }
searchKey
= "[" ("検索" / "Search"i) "]" { return text(); }
/ "検索"
/ "Search"i
// block: codeBlock
codeBlock
= BEGIN "```" lang:$(CHAR*) LF code:codeBlockContent LF "```" END
{
lang = lang.trim();
return CODE_BLOCK(code, lang.length > 0 ? lang : null);
}
codeBlockContent
= (!(LF "```" END) .)+
{ return text(); }
// block: mathBlock
mathBlock
= BEGIN "\\[" LF? formula:mathBlockLines LF? "\\]" END
{
return MATH_BLOCK(formula.trim());
}
mathBlockLines
= mathBlockLine (LF mathBlockLine)*
{ return text(); }
mathBlockLine
= (!"\\]" CHAR)+
// block: center
center
= BEGIN "<center>" LF? content:(!(LF? "</center>" END) i:inline { return i; })+ LF? "</center>" END
{
return CENTER(mergeText(content));
}
//
// inline rules
//
// inline: emoji code
emojiCode
= ":" name:emojiCodeName ":"
{
return EMOJI_CODE(name);
}
emojiCodeName
= [a-z0-9_+-]i+ { return text(); }
// inline: unicode emoji
// NOTE: if the text matches one of the emojis, it will count the length of the emoji sequence and consume it.
unicodeEmoji
= &{ return matchUnicodeEmoji(); } (&{ return consumeDynamically(); } .)+
{
return UNI_EMOJI(text());
}
// inline: big
big
= "***" content:(!"***" i:inline { return i; })+ "***"
{
return FN('tada', { }, mergeText(content));
}
// inline: bold
bold
= "**" content:(!"**" i:inline { return i; })+ "**"
{
return BOLD(mergeText(content));
}
/ "__" content:$(!"__" c:([a-z0-9]i / _) { return c; })+ "__"
{
const parsedContent = applyParser(content, 'inlineParser');
return BOLD(parsedContent);
}
// inline: small
small
= "<small>" content:(!"</small>" i:inline { return i; })+ "</small>"
{
return SMALL(mergeText(content));
}
// inline: italic
italic
= italicTag
/ italicAlt
italicTag
= "<i>" content:(!"</i>" i:inline { return i; })+ "</i>"
{
return ITALIC(mergeText(content));
}
italicAlt
= "*" content:$(!"*" ([a-z0-9]i / _))+ "*" &(EOF / LF / _ / ![a-z0-9]i)
{
const parsedContent = applyParser(content, 'inlineParser');
return ITALIC(parsedContent);
}
/ "_" content:$(!"_" ([a-z0-9]i / _))+ "_" &(EOF / LF / _ / ![a-z0-9]i)
{
const parsedContent = applyParser(content, 'inlineParser');
return ITALIC(parsedContent);
}
// inline: strike
strike
= "~~" content:(!("~" / LF) i:inline { return i; })+ "~~"
{
return STRIKE(mergeText(content));
}
// inline: inlineCode
inlineCode
= "`" content:$(![`´] c:CHAR { return c; })+ "`"
{
return INLINE_CODE(content);
}
// inline: mathInline
mathInline
= "\\(" content:$(!"\\)" c:CHAR { return c; })+ "\\)"
{
return MATH_INLINE(content);
}
// inline: mention
mention
= "@" name:mentionName host:("@" host:mentionHost { return host; })?
{
return MENTION(name, host, text());
}
mentionName
= !"-" mentionNamePart+ // first char is not "-".
{
return text();
}
mentionNamePart
= "-" &mentionNamePart // last char is not "-".
/ [a-z0-9_]i
mentionHost
= ![.-] mentionHostPart+ // first char is neither "." nor "-".
{
return text();
}
mentionHostPart
= [.-] &mentionHostPart // last char is neither "." nor "-".
/ [a-z0-9_]i
// inline: hashtag
hashtag
= "#" !("\uFE0F"? "\u20E3") content:hashtagContent
{
return HASHTAG(content);
}
hashtagContent
= !(invalidHashtagContent !hashtagContentPart) hashtagContentPart+ { return text(); }
invalidHashtagContent
= [0-9]+
hashtagContentPart
= hashtagBracketPair / hashtagChar
hashtagBracketPair
= "(" hashtagContent* ")"
/ "[" hashtagContent* "]"
/ "「" hashtagContent* "」"
hashtagChar
= ![ \t.,!?'"#:\/\[\]【】()「」] CHAR
// inline: URL
url
= "<" url:altUrlFormat ">"
{
return N_URL(url);
}
/ url:urlFormat
{
return N_URL(url);
}
urlFormat
= "http" "s"? "://" urlContentPart+
{
return text();
}
urlContentPart
= urlBracketPair
/ [.,] &urlContentPart // last char is neither "." nor ",".
/ [a-z0-9_/:%#@$&?!~=+-]i
urlBracketPair
= "(" urlContentPart* ")"
/ "[" urlContentPart* "]"
altUrlFormat
= "http" "s"? "://" (!(">" / _) CHAR)+
{
return text();
}
// inline: link
link
= silent:"?"? "[" label:linkLabel "](" url:linkUrl ")"
{
return LINK((silent != null), url, mergeText(label));
}
linkLabel
= parts:linkLabelPart+
{
return parts;
}
linkLabelPart
= url { return text(); /* text node */ }
/ link { return text(); /* text node */ }
/ !"]" n:inline { return n; }
linkUrl
= url { return text(); }
// inline: fn
fnVer1
= "[" name:$([a-z0-9_]i)+ args:fnArgs? _ content:fnContentPart+ "]"
{
args = args || {};
return FN(name, args, mergeText(content));
}
fnVer2
= "$[" name:$([a-z0-9_]i)+ args:fnArgs? _ content:fnContentPart+ "]"
{
args = args || {};
return FN(name, args, mergeText(content));
}
fnArgs
= "." head:fnArg tails:("," arg:fnArg { return arg; })*
{
const args = { };
for (const pair of [head, ...tails]) {
args[pair.k] = pair.v;
}
return args;
}
fnArg
= k:$([a-z0-9_]i)+ "=" v:$([a-z0-9_.]i)+
{
return { k, v };
}
/ k:$([a-z0-9_]i)+
{
return { k: k, v: true };
}
fnContentPart
= !("]") i:inline { return i; }
// inline: text
inlineText
= !(LF / _) [a-z0-9]i &(hashtag / mention / italicAlt) . { return text(); } // hashtag, mention, italic ignore
/ . /* text node */
// inline: text (for plainParser)
plainText
= . /* text node */
//
// General
//
BEGIN "beginning of line"
= LF / &{ return location().start.column == 1; }
END "end of line"
= LF / EOF
EOF
= !.
CHAR
= !LF . { return text(); }
LF
= "\r\n" / [\r\n]
_ "whitespace"
= [ \t\u00a0]
test-d/index.ts
View file @ f5d7cb10
......@@ -4,11 +4,27 @@
*/
import { expectType } from 'tsd';
import { NodeType, MfmUrl } from '../built';
import { NodeType, MfmUrl } from '../src';
import * as P from '../src/internal/core';
describe('#NodeType', () => {
it('returns node that has sprcified type', () => {
test('returns node that has sprcified type', () => {
const x = null as unknown as NodeType<'url'>;
expectType<MfmUrl>(x);
});
});
describe('parser internals', () => {
test('seq', () => {
const first = null as unknown as P.Parser<'first'>;
const second = null as unknown as P.Parser<'second'>;
const third = null as unknown as P.Parser<'third' | 'third-second'>;
expectType<P.Parser<['first', 'second', 'third' | 'third-second']>>(P.seq(first, second, third));
});
test('alt', () => {
const first = null as unknown as P.Parser<'first'>;
const second = null as unknown as P.Parser<'second'>;
const third = null as unknown as P.Parser<'third' | 'third-second'>;
expectType<P.Parser<'first' | 'second' | 'third' | 'third-second'>>(P.alt([first, second, third]));
});
});
test/api.ts
View file @ f5d7cb10
import assert from 'assert';
import * as mfm from '../built/index';
import * as mfm from '../src/index';
import {
TEXT, CENTER, FN, UNI_EMOJI, MENTION, EMOJI_CODE, HASHTAG, N_URL, BOLD, SMALL, ITALIC, STRIKE, QUOTE, MATH_BLOCK, SEARCH, CODE_BLOCK, LINK
} from '../built/index';
} from '../src/index';
describe('API', () => {
describe('toString', () => {
it('basic', () => {
test('basic', () => {
const input =
`before
<center>
......@@ -19,10 +19,155 @@ https://github.com/syuilo/ai
after`;
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
test('single node', () => {
const input = '$[tada Hello]';
assert.strictEqual(mfm.toString(mfm.parse(input)[0]), '$[tada Hello]');
});
test('quote', () => {
const input = `
> abc
>
> 123
`;
assert.strictEqual(mfm.toString(mfm.parse(input)), '> abc\n> \n> 123');
});
test('search', () => {
const input = 'MFM 書き方 123 Search';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
test('block code', () => {
const input = '```\nabc\n```';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
test('math block', () => {
const input = '\\[\ny = 2x + 1\n\\]';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
test('center', () => {
const input = '<center>\nabc\n</center>';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
// test('center (single line)', () => {
// const input = '<center>abc</center>';
// assert.strictEqual(mfm.toString(mfm.parse(input)), input);
// });
test('emoji code', () => {
const input = ':abc:';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
test('unicode emoji', () => {
const input = '今起きた😇';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
test('big', () => {
const input = '***abc***';
const output = '$[tada abc]';
assert.strictEqual(mfm.toString(mfm.parse(input)), output);
});
test('bold', () => {
const input = '**abc**';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
// test('bold tag', () => {
// const input = '<b>abc</b>';
// assert.strictEqual(mfm.toString(mfm.parse(input)), input);
// });
test('small', () => {
const input = '<small>abc</small>';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
// test('italic', () => {
// const input = '*abc*';
// assert.strictEqual(mfm.toString(mfm.parse(input)), input);
// });
test('italic tag', () => {
const input = '<i>abc</i>';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
test('strike', () => {
const input = '~~foo~~';
assert.strictEqual(mfm.toString(mfm.parse(input)), input);
});
// test('strike tag', () => {
// const input = '<s>foo</s>';
// assert.strictEqual(mfm.toString(mfm.parse(input)), input);
// });
test('inline code', () => {
const input = 'AiScript: `#abc = 2`';
assert.strictEqual(mfm.toString(mfm.parse(input)), 'AiScript: `#abc = 2`');
});
test('math inline', () => {
const input = '\\(y = 2x + 3\\)';
assert.strictEqual(mfm.toString(mfm.parse(input)), '\\(y = 2x + 3\\)');
});
test('hashtag', () => {
const input = 'a #misskey b';
assert.strictEqual(mfm.toString(mfm.parse(input)), 'a #misskey b');
});
test('link', () => {
const input = '[Ai](https://github.com/syuilo/ai)';
assert.strictEqual(mfm.toString(mfm.parse(input)), '[Ai](https://github.com/syuilo/ai)');
});
test('silent link', () => {
const input = '?[Ai](https://github.com/syuilo/ai)';
assert.strictEqual(mfm.toString(mfm.parse(input)), '?[Ai](https://github.com/syuilo/ai)');
});
test('fn', () => {
const input = '$[tada Hello]';
assert.strictEqual(mfm.toString(mfm.parse(input)), '$[tada Hello]');
});
test('fn with arguments', () => {
const input = '$[spin.speed=1s,alternate Hello]';
assert.strictEqual(mfm.toString(mfm.parse(input)), '$[spin.speed=1s,alternate Hello]');
});
test('plain', () => {
const input = 'a\n<plain>\nHello\nworld\n</plain>\nb';
assert.strictEqual(mfm.toString(mfm.parse(input)), 'a\n<plain>\nHello\nworld\n</plain>\nb');
});
test('1 line plain', () => {
const input = 'a\n<plain>Hello</plain>\nb';
assert.strictEqual(mfm.toString(mfm.parse(input)), 'a\n<plain>\nHello\n</plain>\nb');
});
test('preserve url brackets', () => {
const input1 = 'https://github.com/syuilo/ai';
assert.strictEqual(mfm.toString(mfm.parse(input1)), input1);
const input2 = '<https://github.com/syuilo/ai>';
assert.strictEqual(mfm.toString(mfm.parse(input2)), input2);
});
});
describe('inspect', () => {
it('replace text', () => {
test('replace text', () => {
const input = 'good morning $[tada everynyan!]';
const result = mfm.parse(input);
mfm.inspect(result, node => {
......@@ -33,7 +178,7 @@ after`;
assert.strictEqual(mfm.toString(result), 'hello $[tada everynyan!]');
});
it('replace text (one item)', () => {
test('replace text (one item)', () => {
const input = 'good morning $[tada everyone!]';
const result = mfm.parse(input);
mfm.inspect(result[1], node => {
......@@ -46,7 +191,7 @@ after`;
});
describe('extract', () => {
it('basic', () => {
test('basic', () => {
const nodes = mfm.parse('@hoge @piyo @bebeyo');
const expect = [
MENTION('hoge', null, '@hoge'),
......@@ -56,7 +201,7 @@ after`;
assert.deepStrictEqual(mfm.extract(nodes, node => node.type == 'mention'), expect);
});
it('nested', () => {
test('nested', () => {
const nodes = mfm.parse('abc:hoge:$[tada 123 @hoge :foo:]:piyo:');
const expect = [
EMOJI_CODE('hoge'),
......
test/core.ts 0 → 100644
View file @ f5d7cb10
import assert from 'assert';
import * as P from '../src/internal/core';
const state = {
nestLimit: 3,
depth: 0,
};
describe('core', () => {
describe('difference', () => {
test('basic', () => {
const parser = P.difference(P.regexp(/\p{Letter}/u), P.str('x'));
let result = parser.handler('x',0,state) as P.Success<any>;
assert.deepStrictEqual(result,P.failure());
result = parser.handler('a',0,state) as P.Success<any>;
assert.deepStrictEqual(result,P.success(1,'a'));
});
test('horizontal whitespace', () => {
const parser = P.difference(P.regexp(/\s/u), P.newline);
let result = parser.handler('\n',0,state) as P.Success<any>;
assert.deepStrictEqual(result,P.failure());
result = parser.handler(' ',0,state) as P.Success<any>;
assert.deepStrictEqual(result,P.success(1,' '));
result = parser.handler('\t',0,state) as P.Success<any>;
assert.deepStrictEqual(result,P.success(1,'\t'));
});
});
});
test/parser.ts
View file @ f5d7cb10
This diff is collapsed.
tsconfig.json
View file @ f5d7cb10
This diff is collapsed.