/* The MIT License (MIT) Copyright (c) 2015-2017 Secret Lab Pty. Ltd. and Yarn Spinner contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ // Comment out to not catch exceptions #define CATCH_EXCEPTIONS using System; using System.Collections.Generic; using Newtonsoft.Json; using Antlr4.Runtime; using Antlr4.Runtime.Misc; using Antlr4.Runtime.Tree; using System.Text; using System.IO; using System.Linq; namespace Yarn { public enum NodeFormat { Unknown, // an unknown type SingleNodeText, // a plain text file containing a single node with no metadata JSON, // a JSON file containing multiple nodes with metadata Text, // a text file containing multiple nodes with metadata } internal class Loader { private Dialogue dialogue; public Program program { get; private set; } // Prints out the list of tokens that the tokeniser found for this node void PrintTokenList(IEnumerable tokenList) { // Sum up the result var sb = new System.Text.StringBuilder(); foreach (var t in tokenList) { sb.AppendLine (string.Format("{0} ({1} line {2})", t.ToString (), t.context, t.lineNumber)); } // Let's see what we got dialogue.LogDebugMessage("Tokens:"); dialogue.LogDebugMessage(sb.ToString()); } // Prints the parse tree for the node void PrintParseTree(Yarn.Parser.ParseNode rootNode) { dialogue.LogDebugMessage("Parse Tree:"); dialogue.LogDebugMessage(rootNode.PrintTree(0)); } // Prepares a loader. 'implementation' is used for logging. public Loader(Dialogue dialogue) { if (dialogue == null) throw new ArgumentNullException ("dialogue"); this.dialogue = dialogue; } // the preprocessor that cleans up things to make it easier on ANTLR // replaces \r\n with \n // adds in INDENTS and DEDENTS where necessary // replaces \t with four spaces // takes in a string of yarn and returns a string the compiler can then use private struct EmissionTuple { public int depth; public bool emitted; public EmissionTuple(int depth, bool emitted) { this.depth = depth; this.emitted = emitted; } } private string preprocessor(string nodeText) { string processed = null; using (StringReader reader = new StringReader(nodeText)) { // a list to hold outputLines once they have been cleaned up List outputLines = new List(); // a stack to keep track of how far indented we are // made up of ints and bools // ints track the depth, bool tracks if we emitted an indent token // starts with 0 and false so we can never fall off the end of the stack Stack indents = new Stack(); indents.Push(new EmissionTuple(0, false)); // a bool to determine if we are in a mode where we need to track indents bool shouldTrackNextIndentation = false; char INDENT = '\a'; char DEDENT = '\v'; //string INDENT = "{"; //string DEDENT = "}"; string OPTION = "->"; string line; while ((line = reader.ReadLine()) != null) { // replacing \t with 4 spaces string tweakedLine = line.Replace("\t", " "); // stripping of any trailing newlines, will add them back in later tweakedLine = tweakedLine.TrimEnd('\r', '\n'); // getting the number of indents on this line int lineIndent = tweakedLine.TakeWhile(Char.IsWhiteSpace).Count(); // working out if it is an option (ie does it start with ->) bool isOption = tweakedLine.TrimStart(' ').StartsWith(OPTION); // are we in a state where we need to track indents? var previous = indents.Peek(); if (shouldTrackNextIndentation && (lineIndent > previous.depth)) { indents.Push(new EmissionTuple(lineIndent, true)); // adding an indent to the stream // tries to add it to the end of the previous line where possible if (outputLines.Count == 0) { tweakedLine = INDENT + tweakedLine; } else { outputLines[outputLines.Count - 1] = outputLines[outputLines.Count - 1] + INDENT; } shouldTrackNextIndentation = false; } // have we finished with the current block of statements else if (lineIndent < previous.depth) { while (lineIndent < indents.Peek().depth) { var topLevel = indents.Pop(); if (topLevel.emitted) { // adding dedents if (outputLines.Count == 0) { tweakedLine = DEDENT + tweakedLine; } else { outputLines[outputLines.Count - 1] = outputLines[outputLines.Count - 1] + DEDENT; } } } } else { shouldTrackNextIndentation = false; } // do we need to track the indents for the next statement? if (isOption) { shouldTrackNextIndentation = true; if (indents.Peek().depth < lineIndent) { indents.Push(new EmissionTuple(lineIndent, false)); } } outputLines.Add(tweakedLine); } // mash it all back together now StringBuilder builder = new StringBuilder(); foreach (string outLine in outputLines) { builder.Append(outLine); builder.Append("\n"); } processed = builder.ToString(); } return processed; } // Given a bunch of raw text, load all nodes that were inside it. // You can call this multiple times to append to the collection of nodes, // but note that new nodes will replace older ones with the same name. // Returns the number of nodes that were loaded. public Program Load(string text, Library library, string fileName, Program includeProgram, bool showTokens, bool showParseTree, string onlyConsiderNode, NodeFormat format, bool experimentalMode = false) { if (format == NodeFormat.Unknown) { format = GetFormatFromFileName(fileName); } // currently experimental node can only be used on yarn.txt yarn files and single nodes if (experimentalMode && (format == NodeFormat.Text || format == NodeFormat.SingleNodeText)) { // this isn't the greatest... if (format == NodeFormat.SingleNodeText) { // it is just the body // need to add a dummy header and body delimiters StringBuilder builder = new StringBuilder(); builder.Append("title:Start\n"); builder.Append("---\n"); builder.Append(text); builder.Append("\n===\n"); text = builder.ToString(); } string inputString = preprocessor(text); ICharStream input = CharStreams.fromstring(inputString); YarnSpinnerLexer lexer = new YarnSpinnerLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); YarnSpinnerParser parser = new YarnSpinnerParser(tokens); // turning off the normal error listener and using ours parser.RemoveErrorListeners(); parser.AddErrorListener(ErrorListener.Instance); IParseTree tree = parser.dialogue(); AntlrCompiler antlrcompiler = new AntlrCompiler(library); antlrcompiler.Compile(tree); // merging in the other program if requested if (includeProgram != null) { antlrcompiler.program.Include(includeProgram); } return antlrcompiler.program; } else { // The final parsed nodes that were in the file we were given Dictionary nodes = new Dictionary(); // Load the raw data and get the array of node title-text pairs var nodeInfos = GetNodesFromText(text, format); int nodesLoaded = 0; foreach (NodeInfo nodeInfo in nodeInfos) { if (onlyConsiderNode != null && nodeInfo.title != onlyConsiderNode) continue; // Attempt to parse every node; log if we encounter any errors #if CATCH_EXCEPTIONS try { #endif if (nodeInfo.title == null) { throw new InvalidOperationException("Tried to load a node with no title."); } if (nodes.ContainsKey(nodeInfo.title)) { throw new InvalidOperationException("Attempted to load a node called " + nodeInfo.title + ", but a node with that name has already been loaded!"); } var lexer = new Lexer(); var tokens = lexer.Tokenise(nodeInfo.title, nodeInfo.body); if (showTokens) PrintTokenList(tokens); var node = new Parser(tokens, library).Parse(); // If this node is tagged "rawText", then preserve its source if (string.IsNullOrEmpty(nodeInfo.tags) == false && nodeInfo.tags.Contains("rawText")) { node.source = nodeInfo.body; } node.name = nodeInfo.title; node.nodeTags = nodeInfo.tagsList; if (showParseTree) PrintParseTree(node); nodes[nodeInfo.title] = node; nodesLoaded++; #if CATCH_EXCEPTIONS } catch (Yarn.TokeniserException t) { // Add file information var message = string.Format("In file {0}: Error reading node {1}: {2}", fileName, nodeInfo.title, t.Message); throw new Yarn.TokeniserException(message); } catch (Yarn.ParseException p) { var message = string.Format("In file {0}: Error parsing node {1}: {2}", fileName, nodeInfo.title, p.Message); throw new Yarn.ParseException(message); } catch (InvalidOperationException e) { var message = string.Format("In file {0}: Error reading node {1}: {2}", fileName, nodeInfo.title, e.Message); throw new InvalidOperationException(message); } #endif } var compiler = new Yarn.Compiler(fileName); foreach (var node in nodes) { compiler.CompileNode(node.Value); } if (includeProgram != null) { compiler.program.Include(includeProgram); } return compiler.program; } } // The raw text of the Yarn node, plus metadata // All properties are serialised except tagsList, which is a derived property [JsonObject(MemberSerialization.OptOut)] public struct NodeInfo { public struct Position { public int x { get; set; } public int y { get; set; } } public string title { get; set; } public string body { get; set; } // The raw "tags" field, containing space-separated tags. This is written // to the file. public string tags { get; set; } public int colorID { get; set; } public Position position { get; set; } // The tags for this node, as a list of individual strings. [JsonIgnore] public List tagsList { get { // If we have no tags list, or it's empty, return the empty list if (tags == null || tags.Length == 0) { return new List(); } return new List(tags.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); } } } internal static NodeFormat GetFormatFromFileName(string fileName) { NodeFormat format; if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) { format = NodeFormat.JSON; } else if (fileName.EndsWith(".yarn.txt", StringComparison.OrdinalIgnoreCase)) { format = NodeFormat.Text; } else if (fileName.EndsWith(".node", StringComparison.OrdinalIgnoreCase)) { format = NodeFormat.SingleNodeText; } else { throw new FormatException(string.Format("Unknown file format for file '{0}'", fileName)); } return format; } // Given either Twine, JSON or XML input, return an array // containing info about the nodes in that file internal NodeInfo[] GetNodesFromText(string text, NodeFormat format) { // All the nodes we found in this file var nodes = new List (); switch (format) { case NodeFormat.SingleNodeText: // If it starts with a comment, treat it as a single-node file var nodeInfo = new NodeInfo(); nodeInfo.title = "Start"; nodeInfo.body = text; nodes.Add(nodeInfo); break; case NodeFormat.JSON: // Parse it as JSON try { nodes = JsonConvert.DeserializeObject>(text); } catch (JsonReaderException e) { dialogue.LogErrorMessage("Error parsing Yarn input: " + e.Message); } break; case NodeFormat.Text: // check for the existence of at least one "---"+newline sentinel, which divides // the headers from the body // we use a regex to match either \r\n or \n line endings if (System.Text.RegularExpressions.Regex.IsMatch(text, "---.?\n") == false) { dialogue.LogErrorMessage("Error parsing input: text appears corrupt (no header sentinel)"); break; } var headerRegex = new System.Text.RegularExpressions.Regex("(?.*): *(?.*)"); var nodeProperties = typeof(NodeInfo).GetProperties(); int lineNumber = 0; using (var reader = new System.IO.StringReader(text)) { string line; while ((line = reader.ReadLine()) != null) { // Create a new node NodeInfo node = new NodeInfo(); // Read header lines do { lineNumber++; // skip empty lines if (line == null || line.Length == 0) { continue; } // Attempt to parse the header var headerMatches = headerRegex.Match(line); if (headerMatches == null) { dialogue.LogErrorMessage(string.Format("Line {0}: Can't parse header '{1}'", lineNumber, line)); continue; } var field = headerMatches.Groups["field"].Value; var value = headerMatches.Groups["value"].Value; // Attempt to set the appropriate property using this field foreach (var property in nodeProperties) { if (property.Name != field) { continue; } // skip properties that can't be written to if (property.CanWrite == false) { continue; } try { var propertyType = property.PropertyType; object convertedValue; if (propertyType.IsAssignableFrom(typeof(string))) { convertedValue = value; } else if (propertyType.IsAssignableFrom(typeof(int))) { convertedValue = int.Parse(value); } else if (propertyType.IsAssignableFrom(typeof(NodeInfo.Position))) { var components = value.Split(','); // we expect 2 components: x and y if (components.Length != 2) { throw new FormatException(); } var position = new NodeInfo.Position(); position.x = int.Parse(components[0]); position.y = int.Parse(components[1]); convertedValue = position; } else { throw new NotSupportedException(); } // we need to box this because structs are value types, // so calling SetValue using 'node' would just modify a copy of 'node' object box = node; property.SetValue(box, convertedValue, null); node = (NodeInfo)box; break; } catch (FormatException) { dialogue.LogErrorMessage(string.Format("{0}: Error setting '{1}': invalid value '{2}'", lineNumber, field, value)); } catch (NotSupportedException) { dialogue.LogErrorMessage(string.Format("{0}: Error setting '{1}': This property cannot be set", lineNumber, field)); } } } while ((line = reader.ReadLine()) != "---"); lineNumber++; // We're past the header; read the body var lines = new List(); // Read header lines until we hit the end of node sentinel or the end of the file while ((line = reader.ReadLine()) != "===" && line != null) { lineNumber++; lines.Add(line); } // We're done reading the lines! Zip 'em up into a string and // store it in the body node.body = string.Join("\n", lines.ToArray()); // And add this node to the list nodes.Add(node); // And now we're ready to move on to the next line! } } break; default: throw new InvalidOperationException("Unknown format " + format.ToString()); } // hooray we're done return nodes.ToArray(); } } }