SyntaxToken.cs 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364
  1. using Lua.Internal;
  2. namespace Lua.CodeAnalysis.Syntax;
  3. public readonly struct SyntaxToken(SyntaxTokenType type, ReadOnlyMemory<char> text, SourcePosition position) : IEquatable<SyntaxToken>
  4. {
  5. public static SyntaxToken EndOfLine(SourcePosition position) => new(SyntaxTokenType.EndOfLine, Keywords.LF.AsMemory(), position);
  6. public static SyntaxToken LParen(SourcePosition position) => new(SyntaxTokenType.LParen, Keywords.LParen.AsMemory(), position);
  7. public static SyntaxToken RParen(SourcePosition position) => new(SyntaxTokenType.RParen, Keywords.RParen.AsMemory(), position);
  8. public static SyntaxToken LCurly(SourcePosition position) => new(SyntaxTokenType.LCurly, Keywords.LCurly.AsMemory(), position);
  9. public static SyntaxToken RCurly(SourcePosition position) => new(SyntaxTokenType.RCurly, Keywords.RCurly.AsMemory(), position);
  10. public static SyntaxToken LSquare(SourcePosition position) => new(SyntaxTokenType.LSquare, Keywords.LSquare.AsMemory(), position);
  11. public static SyntaxToken RSquare(SourcePosition position) => new(SyntaxTokenType.RSquare, Keywords.RSquare.AsMemory(), position);
  12. public static SyntaxToken Nil(SourcePosition position) => new(SyntaxTokenType.Nil, Keywords.Nil.AsMemory(), position);
  13. public static SyntaxToken True(SourcePosition position) => new(SyntaxTokenType.True, Keywords.True.AsMemory(), position);
  14. public static SyntaxToken False(SourcePosition position) => new(SyntaxTokenType.False, Keywords.False.AsMemory(), position);
  15. public static SyntaxToken Addition(SourcePosition position) => new(SyntaxTokenType.Addition, Keywords.Addition.AsMemory(), position);
  16. public static SyntaxToken Subtraction(SourcePosition position) => new(SyntaxTokenType.Subtraction, Keywords.Subtraction.AsMemory(), position);
  17. public static SyntaxToken Multiplication(SourcePosition position) => new(SyntaxTokenType.Multiplication, Keywords.Multiplication.AsMemory(), position);
  18. public static SyntaxToken Division(SourcePosition position) => new(SyntaxTokenType.Division, Keywords.Division.AsMemory(), position);
  19. public static SyntaxToken Modulo(SourcePosition position) => new(SyntaxTokenType.Modulo, Keywords.Modulo.AsMemory(), position);
  20. public static SyntaxToken Exponentiation(SourcePosition position) => new(SyntaxTokenType.Exponentiation, Keywords.Exponentiation.AsMemory(), position);
  21. public static SyntaxToken Equality(SourcePosition position) => new(SyntaxTokenType.Equality, Keywords.Equality.AsMemory(), position);
  22. public static SyntaxToken Inequality(SourcePosition position) => new(SyntaxTokenType.Inequality, Keywords.Inequality.AsMemory(), position);
  23. public static SyntaxToken GreaterThan(SourcePosition position) => new(SyntaxTokenType.GreaterThan, Keywords.GreaterThan.AsMemory(), position);
  24. public static SyntaxToken GreaterThanOrEqual(SourcePosition position) => new(SyntaxTokenType.GreaterThanOrEqual, Keywords.GreaterThanOrEqual.AsMemory(), position);
  25. public static SyntaxToken LessThan(SourcePosition position) => new(SyntaxTokenType.LessThan, Keywords.LessThan.AsMemory(), position);
  26. public static SyntaxToken LessThanOrEqual(SourcePosition position) => new(SyntaxTokenType.LessThanOrEqual, Keywords.LessThanOrEqual.AsMemory(), position);
  27. public static SyntaxToken Length(SourcePosition position) => new(SyntaxTokenType.Length, Keywords.Length.AsMemory(), position);
  28. public static SyntaxToken Concat(SourcePosition position) => new(SyntaxTokenType.Concat, Keywords.Concat.AsMemory(), position);
  29. public static SyntaxToken VarArg(SourcePosition position) => new(SyntaxTokenType.VarArg, "...".AsMemory(), position);
  30. public static SyntaxToken Assignment(SourcePosition position) => new(SyntaxTokenType.Assignment, Keywords.Assignment.AsMemory(), position);
  31. public static SyntaxToken And(SourcePosition position) => new(SyntaxTokenType.And, Keywords.And.AsMemory(), position);
  32. public static SyntaxToken Or(SourcePosition position) => new(SyntaxTokenType.Or, Keywords.Or.AsMemory(), position);
  33. public static SyntaxToken Not(SourcePosition position) => new(SyntaxTokenType.Not, Keywords.Not.AsMemory(), position);
  34. public static SyntaxToken End(SourcePosition position) => new(SyntaxTokenType.End, Keywords.End.AsMemory(), position);
  35. public static SyntaxToken Then(SourcePosition position) => new(SyntaxTokenType.Then, Keywords.Then.AsMemory(), position);
  36. public static SyntaxToken If(SourcePosition position) => new(SyntaxTokenType.If, Keywords.If.AsMemory(), position);
  37. public static SyntaxToken ElseIf(SourcePosition position) => new(SyntaxTokenType.ElseIf, Keywords.ElseIf.AsMemory(), position);
  38. public static SyntaxToken Else(SourcePosition position) => new(SyntaxTokenType.Else, Keywords.Else.AsMemory(), position);
  39. public static SyntaxToken Local(SourcePosition position) => new(SyntaxTokenType.Local, Keywords.Local.AsMemory(), position);
  40. public static SyntaxToken Return(SourcePosition position) => new(SyntaxTokenType.Return, Keywords.Return.AsMemory(), position);
  41. public static SyntaxToken Goto(SourcePosition position) => new(SyntaxTokenType.Goto, Keywords.Goto.AsMemory(), position);
  42. public static SyntaxToken Comma(SourcePosition position) => new(SyntaxTokenType.Comma, ",".AsMemory(), position);
  43. public static SyntaxToken Dot(SourcePosition position) => new(SyntaxTokenType.Dot, ".".AsMemory(), position);
  44. public static SyntaxToken SemiColon(SourcePosition position) => new(SyntaxTokenType.SemiColon, ";".AsMemory(), position);
  45. public static SyntaxToken Colon(SourcePosition position) => new(SyntaxTokenType.Colon, ":".AsMemory(), position);
  46. public static SyntaxToken Do(SourcePosition position) => new(SyntaxTokenType.Do, Keywords.Do.AsMemory(), position);
  47. public static SyntaxToken While(SourcePosition position) => new(SyntaxTokenType.While, Keywords.While.AsMemory(), position);
  48. public static SyntaxToken Repeat(SourcePosition position) => new(SyntaxTokenType.Repeat, Keywords.Repeat.AsMemory(), position);
  49. public static SyntaxToken Until(SourcePosition position) => new(SyntaxTokenType.Until, Keywords.Until.AsMemory(), position);
  50. public static SyntaxToken Break(SourcePosition position) => new(SyntaxTokenType.Break, Keywords.Break.AsMemory(), position);
  51. public static SyntaxToken Function(SourcePosition position) => new(SyntaxTokenType.Function, Keywords.Function.AsMemory(), position);
  52. public static SyntaxToken For(SourcePosition position) => new(SyntaxTokenType.For, Keywords.For.AsMemory(), position);
  53. public static SyntaxToken In(SourcePosition position) => new(SyntaxTokenType.In, Keywords.In.AsMemory(), position);
  54. public SyntaxTokenType Type { get; } = type;
  55. public ReadOnlyMemory<char> Text { get; } = text;
  56. public SourcePosition Position { get; } = position;
  57. public static SyntaxToken Number(string text, SourcePosition position)
  58. {
  59. return new(SyntaxTokenType.Number, text.AsMemory(), position);
  60. }
  61. public static SyntaxToken Number(ReadOnlyMemory<char> text, SourcePosition position)
  62. {
  63. return new(SyntaxTokenType.Number, text, position);
  64. }
  65. public static SyntaxToken Identifier(string text, SourcePosition position)
  66. {
  67. return new(SyntaxTokenType.Identifier, text.AsMemory(), position);
  68. }
  69. public static SyntaxToken Identifier(ReadOnlyMemory<char> text, SourcePosition position)
  70. {
  71. return new(SyntaxTokenType.Identifier, text, position);
  72. }
  73. public static SyntaxToken String(ReadOnlyMemory<char> text, SourcePosition position)
  74. {
  75. return new(SyntaxTokenType.String, text, position);
  76. }
  77. public static SyntaxToken RawString(ReadOnlyMemory<char> text, SourcePosition position)
  78. {
  79. return new(SyntaxTokenType.RawString, text, position);
  80. }
  81. public static SyntaxToken Label(ReadOnlyMemory<char> text, SourcePosition position)
  82. {
  83. return new(SyntaxTokenType.Label, text, position);
  84. }
  85. public override string ToString()
  86. {
  87. return $"{Position} {Type}:{Text}";
  88. }
  89. public string ToDisplayString()
  90. {
  91. return Type switch
  92. {
  93. SyntaxTokenType.EndOfLine => Keywords.LF,
  94. SyntaxTokenType.LParen => Keywords.LParen,
  95. SyntaxTokenType.RParen => Keywords.RParen,
  96. SyntaxTokenType.LCurly => Keywords.LCurly,
  97. SyntaxTokenType.RCurly => Keywords.RCurly,
  98. SyntaxTokenType.LSquare => Keywords.LSquare,
  99. SyntaxTokenType.RSquare => Keywords.RSquare,
  100. SyntaxTokenType.SemiColon => ";",
  101. SyntaxTokenType.Comma => ",",
  102. SyntaxTokenType.Number => Text.ToString(),
  103. SyntaxTokenType.String => $"\"{Text}\"",
  104. SyntaxTokenType.RawString => $"[[{Text}]]",
  105. SyntaxTokenType.Nil => Keywords.Nil,
  106. SyntaxTokenType.True => Keywords.True,
  107. SyntaxTokenType.False => Keywords.False,
  108. SyntaxTokenType.Identifier => Text.ToString(),
  109. SyntaxTokenType.Addition => Keywords.Addition,
  110. SyntaxTokenType.Subtraction => Keywords.Subtraction,
  111. SyntaxTokenType.Multiplication => Keywords.Multiplication,
  112. SyntaxTokenType.Division => Keywords.Division,
  113. SyntaxTokenType.Modulo => Keywords.Modulo,
  114. SyntaxTokenType.Exponentiation => Keywords.Exponentiation,
  115. SyntaxTokenType.Equality => Keywords.Equality,
  116. SyntaxTokenType.Inequality => Keywords.Inequality,
  117. SyntaxTokenType.GreaterThan => Keywords.GreaterThan,
  118. SyntaxTokenType.LessThan => Keywords.LessThan,
  119. SyntaxTokenType.GreaterThanOrEqual => Keywords.GreaterThanOrEqual,
  120. SyntaxTokenType.LessThanOrEqual => Keywords.LessThanOrEqual,
  121. SyntaxTokenType.And => Keywords.And,
  122. SyntaxTokenType.Not => Keywords.Not,
  123. SyntaxTokenType.Or => Keywords.Or,
  124. SyntaxTokenType.Assignment => Keywords.Assignment,
  125. SyntaxTokenType.Concat => Keywords.Concat,
  126. SyntaxTokenType.Length => Keywords.Length,
  127. SyntaxTokenType.Break => Keywords.Break,
  128. SyntaxTokenType.Do => Keywords.Do,
  129. SyntaxTokenType.For => Keywords.For,
  130. SyntaxTokenType.Goto => Keywords.Goto,
  131. SyntaxTokenType.If => Keywords.If,
  132. SyntaxTokenType.ElseIf => Keywords.ElseIf,
  133. SyntaxTokenType.Else => Keywords.Else,
  134. SyntaxTokenType.Function => Keywords.Function,
  135. SyntaxTokenType.End => Keywords.End,
  136. SyntaxTokenType.Then => Keywords.Then,
  137. SyntaxTokenType.In => Keywords.In,
  138. SyntaxTokenType.Local => Keywords.Local,
  139. SyntaxTokenType.Repeat => Keywords.Repeat,
  140. SyntaxTokenType.Return => Keywords.Return,
  141. SyntaxTokenType.Until => Keywords.Until,
  142. SyntaxTokenType.While => Keywords.While,
  143. _ => "",
  144. };
  145. }
  146. public bool Equals(SyntaxToken other)
  147. {
  148. return other.Type == Type &&
  149. other.Text.Span.SequenceEqual(Text.Span) &&
  150. other.Position == Position;
  151. }
  152. public override bool Equals(object? obj)
  153. {
  154. if (obj is SyntaxToken token) return Equals(token);
  155. return false;
  156. }
  157. public override int GetHashCode()
  158. {
  159. return HashCode.Combine(Type, Utf16StringMemoryComparer.Default.GetHashCode(Text), Position);
  160. }
  161. public static bool operator ==(SyntaxToken left, SyntaxToken right)
  162. {
  163. return left.Equals(right);
  164. }
  165. public static bool operator !=(SyntaxToken left, SyntaxToken right)
  166. {
  167. return !(left == right);
  168. }
  169. }
  170. public enum SyntaxTokenType
  171. {
  172. /// <summary>
  173. /// Invalid token
  174. /// </summary>
  175. Invalid,
  176. /// <summary>
  177. /// End of line
  178. /// </summary>
  179. EndOfLine,
  180. /// <summary>
  181. /// Left parenthesis '('
  182. /// </summary>
  183. LParen,
  184. /// <summary>
  185. /// Right parenthesis ')'
  186. /// </summary>
  187. RParen,
  188. /// <summary>
  189. /// Left curly bracket '{'
  190. /// </summary>
  191. LCurly,
  192. /// <summary>
  193. /// Right curly bracket '}'
  194. /// </summary>
  195. RCurly,
  196. /// <summary>
  197. /// Left square bracket '['
  198. /// </summary>
  199. LSquare,
  200. /// <summary>
  201. /// Right square bracket ']'
  202. /// </summary>
  203. RSquare,
  204. /// <summary>
  205. /// Semi colon (;)
  206. /// </summary>
  207. SemiColon,
  208. /// <summary>
  209. /// Colon (:)
  210. /// </summary>
  211. Colon,
  212. /// <summary>
  213. /// Comma (,)
  214. /// </summary>
  215. Comma,
  216. /// <summary>
  217. /// Dot (.)
  218. /// </summary>
  219. Dot,
  220. /// <summary>
  221. /// Numeric literal (e.g. 1, 2, 1.0, 2.0, ...)
  222. /// </summary>
  223. Number,
  224. /// <summary>
  225. /// String literal (e.g. "foo", "bar", ...)
  226. /// </summary>
  227. String,
  228. /// <summary>
  229. /// Raw string literal (e.g. [[Hello, World!]])
  230. /// </summary>
  231. RawString,
  232. /// <summary>
  233. /// Nil literal (nil)
  234. /// </summary>
  235. Nil,
  236. /// <summary>
  237. /// Boolean literal (true)
  238. /// </summary>
  239. True,
  240. /// <summary>
  241. /// Boolean literal (false)
  242. /// </summary>
  243. False,
  244. /// <summary>
  245. /// Identifier
  246. /// </summary>
  247. Identifier,
  248. /// <summary>
  249. /// Label
  250. /// </summary>
  251. Label,
  252. /// <summary>
  253. /// Addition operator (+)
  254. /// </summary>
  255. Addition,
  256. /// <summary>
  257. /// Subtraction operator (-)
  258. /// </summary>
  259. Subtraction,
  260. /// <summary>
  261. /// Multiplication operator (*)
  262. /// </summary>
  263. Multiplication,
  264. /// <summary>
  265. /// Division operator (/)
  266. /// </summary>
  267. Division,
  268. /// <summary>
  269. /// Modulo operator (%)
  270. /// </summary>
  271. Modulo,
  272. /// <summary>
  273. /// Exponentiation operator (^)
  274. /// </summary>
  275. Exponentiation,
  276. Equality, // ==
  277. Inequality, // ~=
  278. GreaterThan, // >
  279. LessThan, // <
  280. GreaterThanOrEqual, // >=
  281. LessThanOrEqual, // <=
  282. And, // and
  283. Not, // not
  284. Or, // or
  285. /// <summary>
  286. /// Assignment operator (=)
  287. /// </summary>
  288. Assignment,
  289. Concat, // ..
  290. Length, // #
  291. VarArg, // ...
  292. Break, // break
  293. Do, // do
  294. For, // for
  295. Goto, // goto
  296. If, // if
  297. ElseIf, // elseif
  298. Else, // else
  299. Function, // function
  300. End, // end
  301. Then, // then
  302. In, // in
  303. Local, // local
  304. Repeat, // repeat
  305. Return, // return
  306. Until, // until
  307. While, // while
  308. }