UnicodeString.hx 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440
  1. /*
  2. * Copyright (C)2005-2019 Haxe Foundation
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20. * DEALINGS IN THE SOFTWARE.
  21. */
  22. import haxe.io.Bytes;
  23. import haxe.io.Encoding;
  24. import haxe.iterators.StringIteratorUnicode;
  25. import haxe.iterators.StringKeyValueIteratorUnicode;
  26. /**
  27. This abstract provides consistent cross-target unicode support for characters of any width.
  28. Due to differing internal representations of strings across targets, only the basic
  29. multilingual plane (BMP) is supported consistently by `String` class.
  30. This abstract provides API to consistently handle all characters even beyond BMP.
  31. @see https://haxe.org/manual/std-String-unicode.html
  32. **/
  33. @:forward
  34. @:access(StringTools)
  35. abstract UnicodeString(String) from String to String {
  36. /**
  37. Tells if `b` is a correctly encoded UTF8 byte sequence.
  38. **/
  39. static public function validate(b:Bytes, encoding:Encoding):Bool {
  40. switch (encoding) {
  41. case RawNative:
  42. throw "UnicodeString.validate: RawNative encoding is not supported";
  43. case UTF8:
  44. var data = b.getData();
  45. var pos = 0;
  46. var max = b.length;
  47. while (pos < max) {
  48. var c:Int = Bytes.fastGet(data, pos++);
  49. if (c < 0x80) {} else if (c < 0xC2) {
  50. return false;
  51. } else if (c < 0xE0) {
  52. if (pos + 1 > max) {
  53. return false;
  54. }
  55. var c2:Int = Bytes.fastGet(data, pos++);
  56. if (c2 < 0x80 || c2 > 0xBF) {
  57. return false;
  58. }
  59. } else if (c < 0xF0) {
  60. if (pos + 2 > max) {
  61. return false;
  62. }
  63. var c2:Int = Bytes.fastGet(data, pos++);
  64. if (c == 0xE0) {
  65. if (c2 < 0xA0 || c2 > 0xBF)
  66. return false;
  67. } else {
  68. if (c2 < 0x80 || c2 > 0xBF)
  69. return false;
  70. }
  71. var c3:Int = Bytes.fastGet(data, pos++);
  72. if (c3 < 0x80 || c3 > 0xBF) {
  73. return false;
  74. }
  75. c = (c << 16) | (c2 << 8) | c3;
  76. if (0xEDA080 <= c && c <= 0xEDBFBF) { // surrogate pairs
  77. return false;
  78. }
  79. } else if (c > 0xF4) {
  80. return false;
  81. } else {
  82. if (pos + 3 > max) {
  83. return false;
  84. }
  85. var c2:Int = Bytes.fastGet(data, pos++);
  86. if (c == 0xF0) {
  87. if (c2 < 0x90 || c2 > 0xBF)
  88. return false;
  89. } else if (c == 0xF4) {
  90. if (c2 < 0x80 || c2 > 0x8F)
  91. return false;
  92. } else {
  93. if (c2 < 0x80 || c2 > 0xBF)
  94. return false;
  95. }
  96. var c3:Int = Bytes.fastGet(data, pos++);
  97. if (c3 < 0x80 || c3 > 0xBF) {
  98. return false;
  99. }
  100. var c4:Int = Bytes.fastGet(data, pos++);
  101. if (c4 < 0x80 || c4 > 0xBF) {
  102. return false;
  103. }
  104. }
  105. }
  106. return true;
  107. }
  108. }
  109. #if target.unicode
  110. /**
  111. Creates an instance of UnicodeString.
  112. **/
  113. public inline function new(string:String):Void {
  114. this = string;
  115. }
  116. /**
  117. Returns an iterator of the unicode code points.
  118. **/
  119. public inline function iterator():StringIteratorUnicode {
  120. return new StringIteratorUnicode(this);
  121. }
  122. /**
  123. Returns an iterator of the code point indices and unicode code points.
  124. **/
  125. public inline function keyValueIterator():StringKeyValueIteratorUnicode {
  126. return new StringKeyValueIteratorUnicode(this);
  127. }
  128. #if target.utf16
  129. /**
  130. The number of characters in `this` String.
  131. **/
  132. public var length(get, never):Int;
  133. /**
  134. Returns the character at position `index` of `this` String.
  135. If `index` is negative or exceeds `this.length`, the empty String `""`
  136. is returned.
  137. **/
  138. public function charAt(index:Int):String {
  139. if (index < 0)
  140. return '';
  141. var unicodeOffset = 0;
  142. var nativeOffset = 0;
  143. while (nativeOffset < this.length) {
  144. var c = StringTools.utf16CodePointAt(this, nativeOffset++);
  145. if (unicodeOffset == index) {
  146. return String.fromCharCode(c);
  147. }
  148. if (c >= StringTools.MIN_SURROGATE_CODE_POINT) {
  149. nativeOffset++;
  150. }
  151. unicodeOffset++;
  152. }
  153. return '';
  154. }
  155. /**
  156. Returns the character code at position `index` of `this` String.
  157. If `index` is negative or exceeds `this.length`, `null` is returned.
  158. **/
  159. public function charCodeAt(index:Int):Null<Int> {
  160. if (index < 0)
  161. return null;
  162. var unicodeOffset = 0;
  163. var nativeOffset = 0;
  164. while (nativeOffset < this.length) {
  165. var c = StringTools.utf16CodePointAt(this, nativeOffset++);
  166. if (unicodeOffset == index) {
  167. return c;
  168. }
  169. if (c >= StringTools.MIN_SURROGATE_CODE_POINT) {
  170. nativeOffset++;
  171. }
  172. unicodeOffset++;
  173. }
  174. return null;
  175. }
  176. /**
  177. @see String.indexOf
  178. **/
  179. public function indexOf(str:String, ?startIndex:Int):Int {
  180. var startIndex:Int = if (startIndex == null || startIndex < 0) {
  181. 0;
  182. } else {
  183. startIndex;
  184. }
  185. if (str.length == 0) {
  186. if (startIndex > length) {
  187. return length;
  188. }
  189. return startIndex;
  190. }
  191. var unicodeOffset = 0;
  192. var nativeOffset = 0;
  193. var matchingOffset = 0;
  194. var result = -1;
  195. while (nativeOffset <= this.length) {
  196. var c = StringTools.utf16CodePointAt(this, nativeOffset);
  197. if (unicodeOffset >= startIndex) {
  198. var c2 = StringTools.utf16CodePointAt(str, matchingOffset);
  199. if (c == c2) {
  200. if (matchingOffset == 0) {
  201. result = unicodeOffset;
  202. }
  203. matchingOffset++;
  204. if (c2 >= StringTools.MIN_SURROGATE_CODE_POINT) {
  205. matchingOffset++;
  206. }
  207. if (matchingOffset == str.length) {
  208. return result;
  209. }
  210. } else if (matchingOffset != 0) {
  211. result = -1;
  212. matchingOffset = 0;
  213. continue;
  214. }
  215. }
  216. nativeOffset++;
  217. if (c >= StringTools.MIN_SURROGATE_CODE_POINT) {
  218. nativeOffset++;
  219. }
  220. unicodeOffset++;
  221. }
  222. return -1;
  223. }
  224. /**
  225. Returns the position of the rightmost occurrence of `str` within `this`
  226. String.
  227. If `startIndex` is given, the search is performed within the substring
  228. of `this` String from 0 to `startIndex + str.length`. Otherwise the search
  229. is performed within `this` String. In either case, the returned position
  230. is relative to the beginning of `this` String.
  231. If `str` cannot be found, -1 is returned.
  232. **/
  233. public function lastIndexOf(str:String, ?startIndex:Int):Int {
  234. if (startIndex == null) {
  235. startIndex = this.length;
  236. } else if (startIndex < 0) {
  237. startIndex = 0;
  238. }
  239. var unicodeOffset = 0;
  240. var nativeOffset = 0;
  241. var result = -1;
  242. var lastIndex = -1;
  243. var matchingOffset = 0;
  244. var strUnicodeLength = (str : UnicodeString).length;
  245. while (nativeOffset < this.length && unicodeOffset < startIndex + strUnicodeLength) {
  246. var c = StringTools.utf16CodePointAt(this, nativeOffset);
  247. var c2 = StringTools.utf16CodePointAt(str, matchingOffset);
  248. if (c == c2) {
  249. if (matchingOffset == 0) {
  250. lastIndex = unicodeOffset;
  251. }
  252. matchingOffset++;
  253. if (c2 >= StringTools.MIN_SURROGATE_CODE_POINT) {
  254. matchingOffset++;
  255. }
  256. if (matchingOffset == str.length) {
  257. result = lastIndex;
  258. lastIndex = -1;
  259. }
  260. } else if (matchingOffset != 0) {
  261. lastIndex = -1;
  262. matchingOffset = 0;
  263. continue;
  264. }
  265. nativeOffset++;
  266. if (c >= StringTools.MIN_SURROGATE_CODE_POINT) {
  267. nativeOffset++;
  268. }
  269. unicodeOffset++;
  270. }
  271. return result;
  272. }
  273. /**
  274. Returns `len` characters of `this` String, starting at position `pos`.
  275. If `len` is omitted, all characters from position `pos` to the end of
  276. `this` String are included.
  277. If `pos` is negative, its value is calculated from the end of `this`
  278. String by `this.length + pos`. If this yields a negative value, 0 is
  279. used instead.
  280. If the calculated position + `len` exceeds `this.length`, the characters
  281. from that position to the end of `this` String are returned.
  282. If `len` is negative, the result is unspecified.
  283. **/
  284. public function substr(pos:Int, ?len:Int):String {
  285. if (pos < 0) {
  286. pos = (this : UnicodeString).length + pos;
  287. if (pos < 0) {
  288. pos = 0;
  289. }
  290. }
  291. if (len != null) {
  292. if (len < 0) {
  293. len = (this : UnicodeString).length + len;
  294. }
  295. if (len <= 0) {
  296. return "";
  297. }
  298. }
  299. var unicodeOffset = 0;
  300. var nativeOffset = 0;
  301. var fromOffset = -1;
  302. var subLength = 0;
  303. while (nativeOffset < this.length) {
  304. var c = StringTools.utf16CodePointAt(this, nativeOffset);
  305. if (unicodeOffset >= pos) {
  306. if (fromOffset < 0) {
  307. if (len == null) {
  308. return this.substr(nativeOffset);
  309. }
  310. fromOffset = nativeOffset;
  311. }
  312. subLength++;
  313. if (subLength >= len) {
  314. var lastOffset = (c < StringTools.MIN_SURROGATE_CODE_POINT ? nativeOffset : nativeOffset + 1);
  315. return this.substr(fromOffset, lastOffset - fromOffset + 1);
  316. }
  317. }
  318. nativeOffset += (c >= StringTools.MIN_SURROGATE_CODE_POINT ? 2 : 1);
  319. unicodeOffset++;
  320. }
  321. return (fromOffset < 0 ? "" : this.substr(fromOffset));
  322. }
  323. /**
  324. Returns the part of `this` String from `startIndex` to but not including `endIndex`.
  325. If `startIndex` or `endIndex` are negative, 0 is used instead.
  326. If `startIndex` exceeds `endIndex`, they are swapped.
  327. If the (possibly swapped) `endIndex` is omitted or exceeds
  328. `this.length`, `this.length` is used instead.
  329. If the (possibly swapped) `startIndex` exceeds `this.length`, the empty
  330. String `""` is returned.
  331. **/
  332. public function substring(startIndex:Int, ?endIndex:Int):String {
  333. if (startIndex < 0) {
  334. startIndex = 0;
  335. }
  336. if (endIndex != null) {
  337. if (endIndex < 0) {
  338. endIndex = 0;
  339. }
  340. if (startIndex == endIndex) {
  341. return "";
  342. }
  343. if (startIndex > endIndex) {
  344. var tmp = startIndex;
  345. startIndex = endIndex;
  346. endIndex = tmp;
  347. }
  348. }
  349. var unicodeOffset = 0;
  350. var nativeOffset = 0;
  351. var fromOffset = -1;
  352. var subLength = 0;
  353. while (nativeOffset < this.length) {
  354. var c = StringTools.utf16CodePointAt(this, nativeOffset);
  355. if (startIndex <= unicodeOffset) {
  356. if (fromOffset < 0) {
  357. if (endIndex == null) {
  358. return this.substr(nativeOffset);
  359. }
  360. fromOffset = nativeOffset;
  361. }
  362. subLength++;
  363. if (subLength >= endIndex - startIndex) {
  364. var lastOffset = (c < StringTools.MIN_SURROGATE_CODE_POINT ? nativeOffset : nativeOffset + 1);
  365. return this.substr(fromOffset, lastOffset - fromOffset + 1);
  366. }
  367. }
  368. nativeOffset += (c >= StringTools.MIN_SURROGATE_CODE_POINT ? 2 : 1);
  369. unicodeOffset++;
  370. }
  371. return (fromOffset < 0 ? "" : this.substr(fromOffset));
  372. }
  373. function get_length():Int {
  374. var l = 0;
  375. for (c in new StringIteratorUnicode(this)) {
  376. l++;
  377. }
  378. return l;
  379. }
  380. #end
  381. #end
  382. @:op(A < B) static function lt(a:UnicodeString, b:UnicodeString):Bool;
  383. @:op(A <= B) static function lte(a:UnicodeString, b:UnicodeString):Bool;
  384. @:op(A > B) static function gt(a:UnicodeString, b:UnicodeString):Bool;
  385. @:op(A >= B) static function gte(a:UnicodeString, b:UnicodeString):Bool;
  386. @:op(A == B) static function eq(a:UnicodeString, b:UnicodeString):Bool;
  387. @:op(A != B) static function neq(a:UnicodeString, b:UnicodeString):Bool;
  388. @:op(A + B) static function add(a:UnicodeString, b:UnicodeString):UnicodeString;
  389. @:op(A += B) static function assignAdd(a:UnicodeString, b:UnicodeString):UnicodeString;
  390. @:op(A + B) @:commutative static function addString(a:UnicodeString, b:String):UnicodeString;
  391. @:op(A += B) @:commutative static function assignAddString(a:UnicodeString, b:String):UnicodeString;
  392. }