strgen.cpp

Go to the documentation of this file.
00001 /* $Id$ */
00002 
00003 /*
00004  * This file is part of OpenTTD.
00005  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
00006  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
00007  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
00008  */
00009 
00012 #include "../stdafx.h"
00013 #include "../core/endian_func.hpp"
00014 #include "../string_func.h"
00015 #include "../strings_type.h"
00016 #include "../language.h"
00017 #include "../misc/getoptdata.h"
00018 #include "../table/control_codes.h"
00019 
00020 #include "strgen.h"
00021 
00022 #include <stdarg.h>
00023 #include <exception>
00024 
00025 #if (!defined(WIN32) && !defined(WIN64)) || defined(__CYGWIN__)
00026 #include <unistd.h>
00027 #include <sys/stat.h>
00028 #endif
00029 
00030 #if defined WIN32 || defined __WATCOMC__
00031 #include <direct.h>
00032 #endif /* WIN32 || __WATCOMC__ */
00033 
00034 #ifdef __MORPHOS__
00035 #ifdef stderr
00036 #undef stderr
00037 #endif
00038 #define stderr stdout
00039 #endif /* __MORPHOS__ */
00040 
00041 #include "../table/strgen_tables.h"
00042 
00043 
00044 #ifdef _MSC_VER
00045 # define LINE_NUM_FMT(s) "%s (%d): warning: %s (" s ")\n"
00046 #else
00047 # define LINE_NUM_FMT(s) "%s:%d: " s ": %s\n"
00048 #endif
00049 
00050 void CDECL strgen_warning(const char *s, ...)
00051 {
00052   char buf[1024];
00053   va_list va;
00054   va_start(va, s);
00055   vsnprintf(buf, lengthof(buf), s, va);
00056   va_end(va);
00057   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, buf);
00058   _warnings++;
00059 }
00060 
00061 void CDECL strgen_error(const char *s, ...)
00062 {
00063   char buf[1024];
00064   va_list va;
00065   va_start(va, s);
00066   vsnprintf(buf, lengthof(buf), s, va);
00067   va_end(va);
00068   fprintf(stderr, LINE_NUM_FMT("error"), _file, _cur_line, buf);
00069   _errors++;
00070 }
00071 
00072 void NORETURN CDECL strgen_fatal(const char *s, ...)
00073 {
00074   char buf[1024];
00075   va_list va;
00076   va_start(va, s);
00077   vsnprintf(buf, lengthof(buf), s, va);
00078   va_end(va);
00079   fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
00080 #ifdef _MSC_VER
00081   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
00082 #endif
00083   throw std::exception();
00084 }
00085 
00086 void NORETURN CDECL error(const char *s, ...)
00087 {
00088   char buf[1024];
00089   va_list va;
00090   va_start(va, s);
00091   vsnprintf(buf, lengthof(buf), s, va);
00092   va_end(va);
00093   fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
00094 #ifdef _MSC_VER
00095   fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
00096 #endif
00097   exit(2);
00098 }
00099 
00101 struct FileStringReader : StringReader {
00102   FILE *fh; 
00103 
00111   FileStringReader(StringData &data, const char *file, bool master, bool translation) :
00112       StringReader(data, file, master, translation)
00113   {
00114     this->fh = fopen(file, "rb");
00115     if (this->fh == NULL) error("Could not open %s", file);
00116   }
00117 
00119   virtual ~FileStringReader()
00120   {
00121     fclose(this->fh);
00122   }
00123 
00124   /* virtual */ char *ReadLine(char *buffer, size_t size)
00125   {
00126     return fgets(buffer, size, this->fh);
00127   }
00128 
00129   /* virtual */ void HandlePragma(char *str);
00130 
00131   /* virtual */ void ParseFile()
00132   {
00133     this->StringReader::ParseFile();
00134 
00135     if (StrEmpty(_lang.name) || StrEmpty(_lang.own_name) || StrEmpty(_lang.isocode)) {
00136       error("Language must include ##name, ##ownname and ##isocode");
00137     }
00138   }
00139 };
00140 
00141 void FileStringReader::HandlePragma(char *str)
00142 {
00143   if (!memcmp(str, "id ", 3)) {
00144     this->data.next_string_id = strtoul(str + 3, NULL, 0);
00145   } else if (!memcmp(str, "name ", 5)) {
00146     strecpy(_lang.name, str + 5, lastof(_lang.name));
00147   } else if (!memcmp(str, "ownname ", 8)) {
00148     strecpy(_lang.own_name, str + 8, lastof(_lang.own_name));
00149   } else if (!memcmp(str, "isocode ", 8)) {
00150     strecpy(_lang.isocode, str + 8, lastof(_lang.isocode));
00151   } else if (!memcmp(str, "plural ", 7)) {
00152     _lang.plural_form = atoi(str + 7);
00153     if (_lang.plural_form >= lengthof(_plural_forms)) {
00154       error("Invalid pluralform %d", _lang.plural_form);
00155     }
00156   } else if (!memcmp(str, "textdir ", 8)) {
00157     if (!memcmp(str + 8, "ltr", 3)) {
00158       _lang.text_dir = TD_LTR;
00159     } else if (!memcmp(str + 8, "rtl", 3)) {
00160       _lang.text_dir = TD_RTL;
00161     } else {
00162       error("Invalid textdir %s", str + 8);
00163     }
00164   } else if (!memcmp(str, "digitsep ", 9)) {
00165     str += 9;
00166     strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator));
00167   } else if (!memcmp(str, "digitsepcur ", 12)) {
00168     str += 12;
00169     strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator_currency));
00170   } else if (!memcmp(str, "decimalsep ", 11)) {
00171     str += 11;
00172     strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_decimal_separator));
00173   } else if (!memcmp(str, "winlangid ", 10)) {
00174     const char *buf = str + 10;
00175     long langid = strtol(buf, NULL, 16);
00176     if (langid > (long)UINT16_MAX || langid < 0) {
00177       error("Invalid winlangid %s", buf);
00178     }
00179     _lang.winlangid = (uint16)langid;
00180   } else if (!memcmp(str, "grflangid ", 10)) {
00181     const char *buf = str + 10;
00182     long langid = strtol(buf, NULL, 16);
00183     if (langid >= 0x7F || langid < 0) {
00184       error("Invalid grflangid %s", buf);
00185     }
00186     _lang.newgrflangid = (uint8)langid;
00187   } else if (!memcmp(str, "gender ", 7)) {
00188     if (this->master) error("Genders are not allowed in the base translation.");
00189     char *buf = str + 7;
00190 
00191     for (;;) {
00192       const char *s = ParseWord(&buf);
00193 
00194       if (s == NULL) break;
00195       if (_lang.num_genders >= MAX_NUM_GENDERS) error("Too many genders, max %d", MAX_NUM_GENDERS);
00196       strecpy(_lang.genders[_lang.num_genders], s, lastof(_lang.genders[_lang.num_genders]));
00197       _lang.num_genders++;
00198     }
00199   } else if (!memcmp(str, "case ", 5)) {
00200     if (this->master) error("Cases are not allowed in the base translation.");
00201     char *buf = str + 5;
00202 
00203     for (;;) {
00204       const char *s = ParseWord(&buf);
00205 
00206       if (s == NULL) break;
00207       if (_lang.num_cases >= MAX_NUM_CASES) error("Too many cases, max %d", MAX_NUM_CASES);
00208       strecpy(_lang.cases[_lang.num_cases], s, lastof(_lang.cases[_lang.num_cases]));
00209       _lang.num_cases++;
00210     }
00211   } else {
00212     error("unknown pragma '%s'", str);
00213   }
00214 }
00215 
00216 bool CompareFiles(const char *n1, const char *n2)
00217 {
00218   FILE *f2 = fopen(n2, "rb");
00219   if (f2 == NULL) return false;
00220 
00221   FILE *f1 = fopen(n1, "rb");
00222   if (f1 == NULL) error("can't open %s", n1);
00223 
00224   size_t l1, l2;
00225   do {
00226     char b1[4096];
00227     char b2[4096];
00228     l1 = fread(b1, 1, sizeof(b1), f1);
00229     l2 = fread(b2, 1, sizeof(b2), f2);
00230 
00231     if (l1 != l2 || memcmp(b1, b2, l1)) {
00232       fclose(f2);
00233       fclose(f1);
00234       return false;
00235     }
00236   } while (l1 != 0);
00237 
00238   fclose(f2);
00239   fclose(f1);
00240   return true;
00241 }
00242 
00244 struct FileWriter {
00245   FILE *fh;             
00246   const char *filename; 
00247 
00252   FileWriter(const char *filename)
00253   {
00254     this->filename = strdup(filename);
00255     this->fh = fopen(this->filename, "wb");
00256 
00257     if (this->fh == NULL) {
00258       error("Could not open %s", this->filename);
00259     }
00260   }
00261 
00263   void Finalise()
00264   {
00265     fclose(this->fh);
00266     this->fh = NULL;
00267   }
00268 
00270   virtual ~FileWriter()
00271   {
00272     /* If we weren't closed an exception was thrown, so remove the termporary file. */
00273     if (fh != NULL) {
00274       fclose(this->fh);
00275       unlink(this->filename);
00276     }
00277     free(this->filename);
00278   }
00279 };
00280 
00281 struct HeaderFileWriter : HeaderWriter, FileWriter {
00283   const char *real_filename;
00285   int prev;
00286 
00291   HeaderFileWriter(const char *filename) : FileWriter("tmp.xxx"),
00292     real_filename(strdup(filename)), prev(0)
00293   {
00294     fprintf(this->fh, "/* This file is automatically generated. Do not modify */\n\n");
00295     fprintf(this->fh, "#ifndef TABLE_STRINGS_H\n");
00296     fprintf(this->fh, "#define TABLE_STRINGS_H\n");
00297   }
00298 
00299   void WriteStringID(const char *name, int stringid)
00300   {
00301     if (prev + 1 != stringid) fprintf(this->fh, "\n");
00302     fprintf(this->fh, "static const StringID %s = 0x%X;\n", name, stringid);
00303     prev = stringid;
00304   }
00305 
00306   void Finalise(const StringData &data)
00307   {
00308     /* Find the plural form with the most amount of cases. */
00309     int max_plural_forms = 0;
00310     for (uint i = 0; i < lengthof(_plural_forms); i++) {
00311       max_plural_forms = max(max_plural_forms, _plural_forms[i].plural_count);
00312     }
00313 
00314     fprintf(this->fh,
00315       "\n"
00316       "static const uint LANGUAGE_PACK_VERSION     = 0x%X;\n"
00317       "static const uint LANGUAGE_MAX_PLURAL       = %d;\n"
00318       "static const uint LANGUAGE_MAX_PLURAL_FORMS = %d;\n\n",
00319       (uint)data.Version(), (uint)lengthof(_plural_forms), max_plural_forms
00320     );
00321 
00322     fprintf(this->fh, "#endif /* TABLE_STRINGS_H */\n");
00323 
00324     this->FileWriter::Finalise();
00325 
00326     if (CompareFiles(this->filename, this->real_filename)) {
00327       /* files are equal. tmp.xxx is not needed */
00328       unlink(this->filename);
00329     } else {
00330       /* else rename tmp.xxx into filename */
00331   #if defined(WIN32) || defined(WIN64)
00332       unlink(this->real_filename);
00333   #endif
00334       if (rename(this->filename, this->real_filename) == -1) error("rename() failed");
00335     }
00336   }
00337 };
00338 
00340 struct LanguageFileWriter : LanguageWriter, FileWriter {
00345   LanguageFileWriter(const char *filename) : FileWriter(filename)
00346   {
00347   }
00348 
00349   void WriteHeader(const LanguagePackHeader *header)
00350   {
00351     this->Write((const byte *)header, sizeof(*header));
00352   }
00353 
00354   void Finalise()
00355   {
00356     fputc(0, this->fh);
00357     this->FileWriter::Finalise();
00358   }
00359 
00360   void Write(const byte *buffer, size_t length)
00361   {
00362     if (fwrite(buffer, sizeof(*buffer), length, this->fh) != length) {
00363       error("Could not write to %s", this->filename);
00364     }
00365   }
00366 };
00367 
00369 static inline void ottd_mkdir(const char *directory)
00370 {
00371 #if defined(WIN32) || defined(__WATCOMC__)
00372     mkdir(directory);
00373 #else
00374     mkdir(directory, 0755);
00375 #endif
00376 }
00377 
00383 static inline char *mkpath(char *buf, size_t buflen, const char *path, const char *file)
00384 {
00385   ttd_strlcpy(buf, path, buflen); // copy directory into buffer
00386 
00387   char *p = strchr(buf, '\0'); // add path seperator if necessary
00388   if (p[-1] != PATHSEPCHAR && (size_t)(p - buf) + 1 < buflen) *p++ = PATHSEPCHAR;
00389   ttd_strlcpy(p, file, buflen - (size_t)(p - buf)); // catenate filename at end of buffer
00390   return buf;
00391 }
00392 
00393 #if defined(__MINGW32__)
00394 
00399 static inline char *replace_pathsep(char *s)
00400 {
00401   for (char *c = s; *c != '\0'; c++) if (*c == '/') *c = '\\';
00402   return s;
00403 }
00404 #else
00405 static inline char *replace_pathsep(char *s) { return s; }
00406 #endif
00407 
00409 static const OptionData _opts[] = {
00410     GETOPT_NOVAL(     'v',  "--version"),
00411   GETOPT_GENERAL('C', '\0', "-export-commands", ODF_NO_VALUE),
00412   GETOPT_GENERAL('L', '\0', "-export-plurals",  ODF_NO_VALUE),
00413   GETOPT_GENERAL('P', '\0', "-export-pragmas",  ODF_NO_VALUE),
00414     GETOPT_NOVAL(     't',  "--todo"),
00415     GETOPT_NOVAL(     'w',  "--warning"),
00416     GETOPT_NOVAL(     'h',  "--help"),
00417   GETOPT_GENERAL('h', '?',  NULL,               ODF_NO_VALUE),
00418     GETOPT_VALUE(     's',  "--source_dir"),
00419     GETOPT_VALUE(     'd',  "--dest_dir"),
00420   GETOPT_END(),
00421 };
00422 
00423 int CDECL main(int argc, char *argv[])
00424 {
00425   char pathbuf[MAX_PATH];
00426   const char *src_dir = ".";
00427   const char *dest_dir = NULL;
00428 
00429   GetOptData mgo(argc - 1, argv + 1, _opts);
00430   for (;;) {
00431     int i = mgo.GetOpt();
00432     if (i == -1) break;
00433 
00434     switch (i) {
00435       case 'v':
00436         puts("$Revision$");
00437         return 0;
00438 
00439       case 'C':
00440         printf("args\tflags\tcommand\treplacement\n");
00441         for (const CmdStruct *cs = _cmd_structs; cs < endof(_cmd_structs); cs++) {
00442           char flags;
00443           switch (cs->value) {
00444             case 0x200E: case 0x200F: // Implicit BIDI controls
00445             case 0x202A: case 0x202B: case 0x202C: case 0x202D: case 0x202E: // Explicit BIDI controls
00446             case 0xA0: // Non breaking space
00447             case '\n': // Newlines may be added too
00448             case '{':  // This special
00449               /* This command may be in the translation when it is not in base */
00450               flags = 'i';
00451               break;
00452 
00453             default:
00454               if (cs->proc == EmitGender) {
00455                 flags = 'g'; // Command needs number of parameters defined by number of genders
00456               } else if (cs->proc == EmitPlural) {
00457                 flags = 'p'; // Command needs number of parameters defined by plural value
00458               } else {
00459                 flags = '0'; // Command needs no parameters
00460               }
00461           }
00462           printf("%i\t%c\t\"%s\"\t\"%s\"\n", cs->consumes, flags, cs->cmd, strstr(cs->cmd, "STRING") ? "STRING" : cs->cmd);
00463         }
00464         return 0;
00465 
00466       case 'L':
00467         printf("count\tdescription\n");
00468         for (const PluralForm *pf = _plural_forms; pf < endof(_plural_forms); pf++) {
00469           printf("%i\t\"%s\"\n", pf->plural_count, pf->description);
00470         }
00471         return 0;
00472 
00473       case 'P':
00474         printf("name\tflags\tdefault\tdescription\n");
00475         for (size_t i = 0; i < lengthof(_pragmas); i++) {
00476           printf("\"%s\"\t%s\t\"%s\"\t\"%s\"\n",
00477               _pragmas[i][0], _pragmas[i][1], _pragmas[i][2], _pragmas[i][3]);
00478         }
00479         return 0;
00480 
00481       case 't':
00482         _show_todo |= 1;
00483         break;
00484 
00485       case 'w':
00486         _show_todo |= 2;
00487         break;
00488 
00489       case 'h':
00490         puts(
00491           "strgen - $Revision$\n"
00492           " -v | --version    print version information and exit\n"
00493           " -t | --todo       replace any untranslated strings with '<TODO>'\n"
00494           " -w | --warning    print a warning for any untranslated strings\n"
00495           " -h | -? | --help  print this help message and exit\n"
00496           " -s | --source_dir search for english.txt in the specified directory\n"
00497           " -d | --dest_dir   put output file in the specified directory, create if needed\n"
00498           " -export-commands  export all commands and exit\n"
00499           " -export-plurals   export all plural forms and exit\n"
00500           " -export-pragmas   export all pragmas and exit\n"
00501           " Run without parameters and strgen will search for english.txt and parse it,\n"
00502           " creating strings.h. Passing an argument, strgen will translate that language\n"
00503           " file using english.txt as a reference and output <language>.lng."
00504         );
00505         return 0;
00506 
00507       case 's':
00508         src_dir = replace_pathsep(mgo.opt);
00509         break;
00510 
00511       case 'd':
00512         dest_dir = replace_pathsep(mgo.opt);
00513         break;
00514 
00515       case -2:
00516         fprintf(stderr, "Invalid arguments\n");
00517         return 0;
00518     }
00519   }
00520 
00521   if (dest_dir == NULL) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
00522 
00523   try {
00524     /* strgen has two modes of operation. If no (free) arguments are passed
00525      * strgen generates strings.h to the destination directory. If it is supplied
00526      * with a (free) parameter the program will translate that language to destination
00527      * directory. As input english.txt is parsed from the source directory */
00528     if (mgo.numleft == 0) {
00529       mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
00530 
00531       /* parse master file */
00532       StringData data(TAB_COUNT);
00533       FileStringReader master_reader(data, pathbuf, true, false);
00534       master_reader.ParseFile();
00535       if (_errors != 0) return 1;
00536 
00537       /* write strings.h */
00538       ottd_mkdir(dest_dir);
00539       mkpath(pathbuf, lengthof(pathbuf), dest_dir, "strings.h");
00540 
00541       HeaderFileWriter writer(pathbuf);
00542       writer.WriteHeader(data);
00543       writer.Finalise(data);
00544     } else if (mgo.numleft >= 1) {
00545       char *r;
00546 
00547       mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
00548 
00549       StringData data(TAB_COUNT);
00550       /* parse master file and check if target file is correct */
00551       FileStringReader master_reader(data, pathbuf, true, false);
00552       master_reader.ParseFile();
00553 
00554       for (int i = 0; i < mgo.numleft; i++) {
00555         data.FreeTranslation();
00556 
00557         const char *translation = replace_pathsep(mgo.argv[i]);
00558         const char *file = strrchr(translation, PATHSEPCHAR);
00559         FileStringReader translation_reader(data, translation, false, file == NULL || strcmp(file + 1, "english.txt") != 0);
00560         translation_reader.ParseFile(); // target file
00561         if (_errors != 0) return 1;
00562 
00563         /* get the targetfile, strip any directories and append to destination path */
00564         r = strrchr(mgo.argv[i], PATHSEPCHAR);
00565         mkpath(pathbuf, lengthof(pathbuf), dest_dir, (r != NULL) ? &r[1] : mgo.argv[i]);
00566 
00567         /* rename the .txt (input-extension) to .lng */
00568         r = strrchr(pathbuf, '.');
00569         if (r == NULL || strcmp(r, ".txt") != 0) r = strchr(pathbuf, '\0');
00570         ttd_strlcpy(r, ".lng", (size_t)(r - pathbuf));
00571 
00572         LanguageFileWriter writer(pathbuf);
00573         writer.WriteLang(data);
00574         writer.Finalise();
00575 
00576         /* if showing warnings, print a summary of the language */
00577         if ((_show_todo & 2) != 0) {
00578           fprintf(stdout, "%d warnings and %d errors for %s\n", _warnings, _errors, pathbuf);
00579         }
00580       }
00581     }
00582   } catch (...) {
00583     return 2;
00584   }
00585 
00586   return 0;
00587 }