OpenTTD

- Main Page
- Related Pages
- Modules
- Namespaces
- Data Structures
- Files

- File List
- Globals

string.cpp
Go to the documentation of this file.
1 /* $Id: string.cpp 27380 2015-08-10 20:21:29Z michi_cc $ */
2
3 /*
4 * This file is part of OpenTTD.
5 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
6 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
7 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
8 */
9
12 #include "stdafx.h"
13 #include "debug.h"
14 #include "core/alloc_func.hpp"
15 #include "core/math_func.hpp"
16 #include "string_func.h"
17 #include "string_base.h"
18
19 #include "table/control_codes.h"
20
21 #include <stdarg.h>
22 #include <ctype.h> /* required for tolower() */
23
24 #ifdef _MSC_VER
25 #include <errno.h> // required by vsnprintf implementation for MSVC
26 #endif
27
28 #ifdef WITH_ICU_SORT
29 /* Required by strnatcmp. */
30 #include <unicode/ustring.h>
31 #include "language.h"
32 #include "gfx_func.h"
33 #endif /* WITH_ICU_SORT */
34
35 /* The function vsnprintf is used internally to perform the required formatting
36 * tasks. As such this one must be allowed, and makes sure it's terminated. */
37 #include "safeguards.h"
38 #undef vsnprintf
39
50 int CDECL vseprintf(char *str, const char *last, const char *format, va_list ap)
51 {
52 ptrdiff_t diff = last - str;
53 if (diff < 0) return 0;
54 return min((int)diff, vsnprintf(str, diff + 1, format, ap));
55 }
56
73 char *strecat(char *dst, const char *src, const char *last)
74 {
75 assert(dst <= last);
76 while (*dst != '\0') {
77 if (dst == last) return dst;
78 dst++;
79 }
80
81 return strecpy(dst, src, last);
82 }
83
84
101 char *strecpy(char *dst, const char *src, const char *last)
102 {
103 assert(dst <= last);
104 while (dst != last && *src != '\0') {
105 *dst++ = *src++;
106 }
107 *dst = '\0';
108
109 if (dst == last && *src != '\0') {
110 #if defined(STRGEN) || defined(SETTINGSGEN)
111 error("String too long for destination buffer");
112 #else /* STRGEN || SETTINGSGEN */
113 DEBUG(misc, 0, "String too long for destination buffer");
114 #endif /* STRGEN || SETTINGSGEN */
115 }
116 return dst;
117 }
118
126 char *stredup(const char *s, const char *last)
127 {
128 size_t len = last == NULL ? strlen(s) : ttd_strnlen(s, last - s + 1);
129 char *tmp = CallocT<char>(len + 1);
130 memcpy(tmp, s, len);
131 return tmp;
132 }
133
139 char *CDECL str_fmt(const char *str, ...)
140 {
141 char buf[4096];
142 va_list va;
143
144 va_start(va, str);
145 int len = vseprintf(buf, lastof(buf), str, va);
146 va_end(va);
147 char *p = MallocT<char>(len + 1);
148 memcpy(p, buf, len + 1);
149 return p;
150 }
151
158 void str_fix_scc_encoded(char *str, const char *last)
159 {
160 while (str <= last && *str != '\0') {
161 size_t len = Utf8EncodedCharLen(*str);
162 if ((len == 0 && str + 4 > last) || str + len > last) break;
163
164 WChar c;
165 len = Utf8Decode(&c, str);
166 if (c == '\0') break;
167
168 if (c == 0xE028 || c == 0xE02A) {
169 c = SCC_ENCODED;
170 }
171 str += Utf8Encode(str, c);
172 }
173 *str = '\0';
174 }
175
176
184 void str_validate(char *str, const char *last, StringValidationSettings settings)
185 {
186 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
187
188 char *dst = str;
189 while (str <= last && *str != '\0') {
190 size_t len = Utf8EncodedCharLen(*str);
191 /* If the character is unknown, i.e. encoded length is 0
192 * we assume worst case for the length check.
193 * The length check is needed to prevent Utf8Decode to read
194 * over the terminating '\0' if that happens to be placed
195 * within the encoding of an UTF8 character. */
196 if ((len == 0 && str + 4 > last) || str + len > last) break;
197
198 WChar c;
199 len = Utf8Decode(&c, str);
200 /* It's possible to encode the string termination character
201 * into a multiple bytes. This prevents those termination
202 * characters to be skipped */
203 if (c == '\0') break;
204
205 if ((IsPrintable(c) && (c < SCC_SPRITE_START || c > SCC_SPRITE_END)) || ((settings & SVS_ALLOW_CONTROL_CODE) != 0 && c == SCC_ENCODED)) {
206 /* Copy the character back. Even if dst is current the same as str
207 * (i.e. no characters have been changed) this is quicker than
208 * moving the pointers ahead by len */
209 do {
210 *dst++ = *str++;
211 } while (--len != 0);
212 } else if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\n') {
213 *dst++ = *str++;
214 } else {
215 if ((settings & SVS_ALLOW_NEWLINE) != 0 && c == '\r' && str[1] == '\n') {
216 str += len;
217 continue;
218 }
219 /* Replace the undesirable character with a question mark */
220 str += len;
221 if ((settings & SVS_REPLACE_WITH_QUESTION_MARK) != 0) *dst++ = '?';
222 }
223 }
224
225 *dst = '\0';
226 }
227
233 void ValidateString(const char *str)
234 {
235 /* We know it is '\0' terminated. */
236 str_validate(const_cast<char *>(str), str + strlen(str) + 1);
237 }
238
239
247 bool StrValid(const char *str, const char *last)
248 {
249 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
250
251 while (str <= last && *str != '\0') {
252 size_t len = Utf8EncodedCharLen(*str);
253 /* Encoded length is 0 if the character isn't known.
254 * The length check is needed to prevent Utf8Decode to read
255 * over the terminating '\0' if that happens to be placed
256 * within the encoding of an UTF8 character. */
257 if (len == 0 || str + len > last) return false;
258
259 WChar c;
260 len = Utf8Decode(&c, str);
261 if (!IsPrintable(c) || (c >= SCC_SPRITE_START && c <= SCC_SPRITE_END)) {
262 return false;
263 }
264
265 str += len;
266 }
267
268 return *str == '\0';
269 }
270
272 void str_strip_colours(char *str)
273 {
274 char *dst = str;
275 WChar c;
276 size_t len;
277
278 for (len = Utf8Decode(&c, str); c != '\0'; len = Utf8Decode(&c, str)) {
279 if (c < SCC_BLUE || c > SCC_BLACK) {
280 /* Copy the character back. Even if dst is current the same as str
281 * (i.e. no characters have been changed) this is quicker than
282 * moving the pointers ahead by len */
283 do {
284 *dst++ = *str++;
285 } while (--len != 0);
286 } else {
287 /* Just skip (strip) the colour codes */
288 str += len;
289 }
290 }
291 *dst = '\0';
292 }
293
300 size_t Utf8StringLength(const char *s)
301 {
302 size_t len = 0;
303 const char *t = s;
304 while (Utf8Consume(&t) != 0) len++;
305 return len;
306 }
307
308
320 bool strtolower(char *str)
321 {
322 bool changed = false;
323 for (; *str != '\0'; str++) {
324 char new_str = tolower(*str);
325 changed |= new_str != *str;
326 *str = new_str;
327 }
328 return changed;
329 }
330
338 bool IsValidChar(WChar key, CharSetFilter afilter)
339 {
340 switch (afilter) {
341 case CS_ALPHANUMERAL: return IsPrintable(key);
342 case CS_NUMERAL: return (key >= '0' && key <= '9');
343 case CS_NUMERAL_SPACE: return (key >= '0' && key <= '9') || key == ' ';
344 case CS_ALPHA: return IsPrintable(key) && !(key >= '0' && key <= '9');
345 case CS_HEXADECIMAL: return (key >= '0' && key <= '9') || (key >= 'a' && key <= 'f') || (key >= 'A' && key <= 'F');
346 }
347
348 return false;
349 }
350
351 #ifdef WIN32
352 #if defined(_MSC_VER) && _MSC_VER < 1900
353
360 int CDECL vsnprintf(char *str, size_t size, const char *format, va_list ap)
361 {
362 if (size == 0) return 0;
363
364 errno = 0;
365 int ret = _vsnprintf(str, size, format, ap);
366
367 if (ret < 0) {
368 if (errno != ERANGE) {
369 /* There's a formatting error, better get that looked
370 * at properly instead of ignoring it. */
371 NOT_REACHED();
372 }
373 } else if ((size_t)ret < size) {
374 /* The buffer is big enough for the number of
375 * characters stored (excluding null), i.e.
376 * the string has been null-terminated. */
377 return ret;
378 }
379
380 /* The buffer is too small for _vsnprintf to write the
381 * null-terminator at its end and return size. */
382 str[size - 1] = '\0';
383 return (int)size;
384 }
385 #endif /* _MSC_VER */
386
387 #endif /* WIN32 */
388
398 int CDECL seprintf(char *str, const char *last, const char *format, ...)
399 {
400 va_list ap;
401
402 va_start(ap, format);
403 int ret = vseprintf(str, last, format, ap);
404 va_end(ap);
405 return ret;
406 }
407
408
416 char *md5sumToString(char *buf, const char *last, const uint8 md5sum[16])
417 {
418 char *p = buf;
419
420 for (uint i = 0; i < 16; i++) {
421 p += seprintf(p, last, "%02X", md5sum[i]);
422 }
423
424 return p;
425 }
426
427
428 /* UTF-8 handling routines */
429
430
437 size_t Utf8Decode(WChar *c, const char *s)
438 {
439 assert(c != NULL);
440
441 if (!HasBit(s[0], 7)) {
442 /* Single byte character: 0xxxxxxx */
443 *c = s[0];
444 return 1;
445 } else if (GB(s[0], 5, 3) == 6) {
446 if (IsUtf8Part(s[1])) {
447 /* Double byte character: 110xxxxx 10xxxxxx */
448 *c = GB(s[0], 0, 5) << 6 | GB(s[1], 0, 6);
449 if (*c >= 0x80) return 2;
450 }
451 } else if (GB(s[0], 4, 4) == 14) {
452 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2])) {
453 /* Triple byte character: 1110xxxx 10xxxxxx 10xxxxxx */
454 *c = GB(s[0], 0, 4) << 12 | GB(s[1], 0, 6) << 6 | GB(s[2], 0, 6);
455 if (*c >= 0x800) return 3;
456 }
457 } else if (GB(s[0], 3, 5) == 30) {
458 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2]) && IsUtf8Part(s[3])) {
459 /* 4 byte character: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
460 *c = GB(s[0], 0, 3) << 18 | GB(s[1], 0, 6) << 12 | GB(s[2], 0, 6) << 6 | GB(s[3], 0, 6);
461 if (*c >= 0x10000 && *c <= 0x10FFFF) return 4;
462 }
463 }
464
465 /* DEBUG(misc, 1, "[utf8] invalid UTF-8 sequence"); */
466 *c = '?';
467 return 1;
468 }
469
470
477 size_t Utf8Encode(char *buf, WChar c)
478 {
479 if (c < 0x80) {
480 *buf = c;
481 return 1;
482 } else if (c < 0x800) {
483 *buf++ = 0xC0 + GB(c, 6, 5);
484 *buf = 0x80 + GB(c, 0, 6);
485 return 2;
486 } else if (c < 0x10000) {
487 *buf++ = 0xE0 + GB(c, 12, 4);
488 *buf++ = 0x80 + GB(c, 6, 6);
489 *buf = 0x80 + GB(c, 0, 6);
490 return 3;
491 } else if (c < 0x110000) {
492 *buf++ = 0xF0 + GB(c, 18, 3);
493 *buf++ = 0x80 + GB(c, 12, 6);
494 *buf++ = 0x80 + GB(c, 6, 6);
495 *buf = 0x80 + GB(c, 0, 6);
496 return 4;
497 }
498
499 /* DEBUG(misc, 1, "[utf8] can't UTF-8 encode value 0x%X", c); */
500 *buf = '?';
501 return 1;
502 }
503
511 size_t Utf8TrimString(char *s, size_t maxlen)
512 {
513 size_t length = 0;
514
515 for (const char *ptr = strchr(s, '\0'); *s != '\0';) {
516 size_t len = Utf8EncodedCharLen(*s);
517 /* Silently ignore invalid UTF8 sequences, our only concern trimming */
518 if (len == 0) len = 1;
519
520 /* Take care when a hard cutoff was made for the string and
521 * the last UTF8 sequence is invalid */
522 if (length + len >= maxlen || (s + len > ptr)) break;
523 s += len;
524 length += len;
525 }
526
527 *s = '\0';
528 return length;
529 }
530
531 #ifdef DEFINE_STRCASESTR
532 char *strcasestr(const char *haystack, const char *needle)
533 {
534 size_t hay_len = strlen(haystack);
535 size_t needle_len = strlen(needle);
536 while (hay_len >= needle_len) {
537 if (strncasecmp(haystack, needle, needle_len) == 0) return const_cast<char *>(haystack);
538
539 haystack++;
540 hay_len--;
541 }
542
543 return NULL;
544 }
545 #endif /* DEFINE_STRCASESTR */
546
555 static const char *SkipGarbage(const char *str)
556 {
557 while (*str != '\0' && (*str < '0' || IsInsideMM(*str, ';', '@' + 1) || IsInsideMM(*str, '[', '`' + 1) || IsInsideMM(*str, '{', '~' + 1))) str++;
558 return str;
559 }
560
569 int strnatcmp(const char *s1, const char *s2, bool ignore_garbage_at_front)
570 {
571 if (ignore_garbage_at_front) {
572 s1 = SkipGarbage(s1);
573 s2 = SkipGarbage(s2);
574 }
575 #ifdef WITH_ICU_SORT
576 if (_current_collator != NULL) {
577 UErrorCode status = U_ZERO_ERROR;
578 int result = _current_collator->compareUTF8(s1, s2, status);
579 if (U_SUCCESS(status)) return result;
580 }
581
582 #endif /* WITH_ICU_SORT */
583
584 /* Do a normal comparison if ICU is missing or if we cannot create a collator. */
585 return strcasecmp(s1, s2);
586 }
587
588 #ifdef WITH_ICU_SORT
589
590 #include <unicode/utext.h>
591 #include <unicode/brkiter.h>
592
594 class IcuStringIterator : public StringIterator
595 {
596 icu::BreakIterator *char_itr;
597 icu::BreakIterator *word_itr;
598
599 SmallVector<UChar, 32> utf16_str;
600 SmallVector<size_t, 32> utf16_to_utf8;
601
602 public:
603 IcuStringIterator() : char_itr(NULL), word_itr(NULL)
604 {
605 UErrorCode status = U_ZERO_ERROR;
606 this->char_itr = icu::BreakIterator::createCharacterInstance(icu::Locale(_current_language != NULL ? _current_language->isocode : "en"), status);
607 this->word_itr = icu::BreakIterator::createWordInstance(icu::Locale(_current_language != NULL ? _current_language->isocode : "en"), status);
608
609 *this->utf16_str.Append() = '\0';
610 *this->utf16_to_utf8.Append() = 0;
611 }
612
613 virtual ~IcuStringIterator()
614 {
615 delete this->char_itr;
616 delete this->word_itr;
617 }
618
619 virtual void SetString(const char *s)
620 {
621 const char *string_base = s;
622
623 /* Unfortunately current ICU versions only provide rudimentary support
624 * for word break iterators (especially for CJK languages) in combination
625 * with UTF-8 input. As a work around we have to convert the input to
626 * UTF-16 and create a mapping back to UTF-8 character indices. */
627 this->utf16_str.Clear();
628 this->utf16_to_utf8.Clear();
629
630 while (*s != '\0') {
631 size_t idx = s - string_base;
632
633 WChar c = Utf8Consume(&s);
634 if (c < 0x10000) {
635 *this->utf16_str.Append() = (UChar)c;
636 } else {
637 /* Make a surrogate pair. */
638 *this->utf16_str.Append() = (UChar)(0xD800 + ((c - 0x10000) >> 10));
639 *this->utf16_str.Append() = (UChar)(0xDC00 + ((c - 0x10000) & 0x3FF));
640 *this->utf16_to_utf8.Append() = idx;
641 }
642 *this->utf16_to_utf8.Append() = idx;
643 }
644 *this->utf16_str.Append() = '\0';
645 *this->utf16_to_utf8.Append() = s - string_base;
646
647 UText text = UTEXT_INITIALIZER;
648 UErrorCode status = U_ZERO_ERROR;
649 utext_openUChars(&text, this->utf16_str.Begin(), this->utf16_str.Length() - 1, &status);
650 this->char_itr->setText(&text, status);
651 this->word_itr->setText(&text, status);
652 this->char_itr->first();
653 this->word_itr->first();
654 }
655
656 virtual size_t SetCurPosition(size_t pos)
657 {
658 /* Convert incoming position to an UTF-16 string index. */
659 uint utf16_pos = 0;
660 for (uint i = 0; i < this->utf16_to_utf8.Length(); i++) {
661 if (this->utf16_to_utf8[i] == pos) {
662 utf16_pos = i;
663 break;
664 }
665 }
666
667 /* isBoundary has the documented side-effect of setting the current
668 * position to the first valid boundary equal to or greater than
669 * the passed value. */
670 this->char_itr->isBoundary(utf16_pos);
671 return this->utf16_to_utf8[this->char_itr->current()];
672 }
673
674 virtual size_t Next(IterType what)
675 {
676 int32_t pos;
677 switch (what) {
678 case ITER_CHARACTER:
679 pos = this->char_itr->next();
680 break;
681
682 case ITER_WORD:
683 pos = this->word_itr->following(this->char_itr->current());
684 /* The ICU word iterator considers both the start and the end of a word a valid
685 * break point, but we only want word starts. Move to the next location in
686 * case the new position points to whitespace. */
687 while (pos != icu::BreakIterator::DONE &&
688 IsWhitespace(Utf16DecodeChar((const uint16 *)&this->utf16_str[pos]))) {
689 int32_t new_pos = this->word_itr->next();
690 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
691 * even though the iterator wasn't at the end of the string before. */
692 if (new_pos == icu::BreakIterator::DONE) break;
693 pos = new_pos;
694 }
695
696 this->char_itr->isBoundary(pos);
697 break;
698
699 default:
700 NOT_REACHED();
701 }
702
703 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
704 }
705
706 virtual size_t Prev(IterType what)
707 {
708 int32_t pos;
709 switch (what) {
710 case ITER_CHARACTER:
711 pos = this->char_itr->previous();
712 break;
713
714 case ITER_WORD:
715 pos = this->word_itr->preceding(this->char_itr->current());
716 /* The ICU word iterator considers both the start and the end of a word a valid
717 * break point, but we only want word starts. Move to the previous location in
718 * case the new position points to whitespace. */
719 while (pos != icu::BreakIterator::DONE &&
720 IsWhitespace(Utf16DecodeChar((const uint16 *)&this->utf16_str[pos]))) {
721 int32_t new_pos = this->word_itr->previous();
722 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
723 * even though the iterator wasn't at the start of the string before. */
724 if (new_pos == icu::BreakIterator::DONE) break;
725 pos = new_pos;
726 }
727
728 this->char_itr->isBoundary(pos);
729 break;
730
731 default:
732 NOT_REACHED();
733 }
734
735 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
736 }
737 };
738
739 /* static */ StringIterator *StringIterator::Create()
740 {
741 return new IcuStringIterator();
742 }
743
744 #else
745
747 class DefaultStringIterator : public StringIterator
748 {
749 const char *string;
750 size_t len;
751 size_t cur_pos;
752
753 public:
754 DefaultStringIterator() : string(NULL), len(0), cur_pos(0)
755 {
756 }
757
758 virtual void SetString(const char *s)
759 {
760 this->string = s;
761 this->len = strlen(s);
762 this->cur_pos = 0;
763 }
764
765 virtual size_t SetCurPosition(size_t pos)
766 {
767 assert(this->string != NULL && pos <= this->len);
768 /* Sanitize in case we get a position inside an UTF-8 sequence. */
769 while (pos > 0 && IsUtf8Part(this->string[pos])) pos--;
770 return this->cur_pos = pos;
771 }
772
773 virtual size_t Next(IterType what)
774 {
775 assert(this->string != NULL);
776
777 /* Already at the end? */
778 if (this->cur_pos >= this->len) return END;
779
780 switch (what) {
781 case ITER_CHARACTER: {
782 WChar c;
783 this->cur_pos += Utf8Decode(&c, this->string + this->cur_pos);
784 return this->cur_pos;
785 }
786
787 case ITER_WORD: {
788 WChar c;
789 /* Consume current word. */
790 size_t offs = Utf8Decode(&c, this->string + this->cur_pos);
791 while (this->cur_pos < this->len && !IsWhitespace(c)) {
792 this->cur_pos += offs;
793 offs = Utf8Decode(&c, this->string + this->cur_pos);
794 }
795 /* Consume whitespace to the next word. */
796 while (this->cur_pos < this->len && IsWhitespace(c)) {
797 this->cur_pos += offs;
798 offs = Utf8Decode(&c, this->string + this->cur_pos);
799 }
800
801 return this->cur_pos;
802 }
803
804 default:
805 NOT_REACHED();
806 }
807
808 return END;
809 }
810
811 virtual size_t Prev(IterType what)
812 {
813 assert(this->string != NULL);
814
815 /* Already at the beginning? */
816 if (this->cur_pos == 0) return END;
817
818 switch (what) {
819 case ITER_CHARACTER:
820 return this->cur_pos = Utf8PrevChar(this->string + this->cur_pos) - this->string;
821
822 case ITER_WORD: {
823 const char *s = this->string + this->cur_pos;
824 WChar c;
825 /* Consume preceding whitespace. */
826 do {
827 s = Utf8PrevChar(s);
828 Utf8Decode(&c, s);
829 } while (s > this->string && IsWhitespace(c));
830 /* Consume preceding word. */
831 while (s > this->string && !IsWhitespace(c)) {
832 s = Utf8PrevChar(s);
833 Utf8Decode(&c, s);
834 }
835 /* Move caret back to the beginning of the word. */
836 if (IsWhitespace(c)) Utf8Consume(&s);
837
838 return this->cur_pos = s - this->string;
839 }
840
841 default:
842 NOT_REACHED();
843 }
844
845 return END;
846 }
847 };
848
849 /* static */ StringIterator *StringIterator::Create()
850 {
851 return new DefaultStringIterator();
852 }
853
854 #endif