[llvm-rc] Serialize HTML resources to .res files (serialization, pt 1).

This allows to process HTML resources defined in .rc scripts and output
them to resulting .res files. Additionally, some infrastructure allowing
to output these files is created.

This is the first resource type we can operate on.

Thanks to Nico Weber for his original work in this area.

Differential Revision: reviews.llvm.org/D37283

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@314538 91177308-0d34-0410-b5e6-96231b3b80d8

7 files changed, 501 insertions, 5 deletions

diff --git a/tools/llvm-rc/CMakeLists.txt b/tools/llvm-rc/CMakeLists.txt
index dce6cbb1121..e5c0eb25d7b 100644
--- a/tools/llvm-rc/CMakeLists.txt
+++ b/tools/llvm-rc/CMakeLists.txt
@@ -10,6 +10,7 @@ add_public_tablegen_target(RcTableGen)
 
 add_llvm_tool(llvm-rc
   llvm-rc.cpp
+  ResourceFileWriter.cpp
   ResourceScriptParser.cpp
   ResourceScriptStmt.cpp
   ResourceScriptToken.cpp
diff --git a/tools/llvm-rc/Opts.td b/tools/llvm-rc/Opts.td
index 4f6bf27e8d5..9792aa582cb 100644
--- a/tools/llvm-rc/Opts.td
+++ b/tools/llvm-rc/Opts.td
@@ -32,6 +32,9 @@ def H : Flag<[ "/", "-" ], "H">,
         Alias<HELP>,
         HelpText<"Display this help and exit.">;
 
+def DRY_RUN : Flag<[ "/", "-" ], "dry-run">,
+              HelpText<"Don't compile the input; only try to parse it.">;
+
 // Unused switches (at least for now). These will stay unimplemented
 // in an early stage of development and can be ignored. However, we need to
 // parse them in order to preserve the compatibility with the original tool.
diff --git a/tools/llvm-rc/ResourceFileWriter.cpp b/tools/llvm-rc/ResourceFileWriter.cpp
new file mode 100644
index 00000000000..642d6ef377c
--- /dev/null
+++ b/tools/llvm-rc/ResourceFileWriter.cpp
@@ -0,0 +1,241 @@
+//===-- ResourceFileWriter.cpp --------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This implements the visitor serializing resources to a .res stream.
+//
+//===---------------------------------------------------------------------===//
+
+#include "ResourceFileWriter.h"
+
+#include "llvm/Object/WindowsResource.h"
+#include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/EndianStream.h"
+
+using namespace llvm::support;
+
+// Take an expression returning llvm::Error and forward the error if it exists.
+#define RETURN_IF_ERROR(Expr)                                                  \
+  if (auto Err = (Expr))                                                       \
+    return Err;
+
+namespace llvm {
+namespace rc {
+
+static Error createError(Twine Message,
+                         std::errc Type = std::errc::invalid_argument) {
+  return make_error<StringError>(Message, std::make_error_code(Type));
+}
+
+static Error checkNumberFits(uint32_t Number, size_t MaxBits, Twine FieldName) {
+  assert(1 <= MaxBits && MaxBits <= 32);
+  if (!(Number >> MaxBits))
+    return Error::success();
+  return createError(FieldName + " (" + Twine(Number) + ") does not fit in " +
+                         Twine(MaxBits) + " bits.",
+                     std::errc::value_too_large);
+}
+
+template <typename FitType>
+static Error checkNumberFits(uint32_t Number, Twine FieldName) {
+  return checkNumberFits(Number, sizeof(FitType) * 8, FieldName);
+}
+
+static Error checkIntOrString(IntOrString Value, Twine FieldName) {
+  if (!Value.isInt())
+    return Error::success();
+  return checkNumberFits<uint16_t>(Value.getInt(), FieldName);
+}
+
+static bool stripQuotes(StringRef &Str, bool &IsLongString) {
+  if (!Str.contains('"'))
+    return false;
+
+  // Just take the contents of the string, checking if it's been marked long.
+  IsLongString = Str.startswith_lower("L");
+  if (IsLongString)
+    Str = Str.drop_front();
+
+  bool StripSuccess = Str.consume_front("\"") && Str.consume_back("\"");
+  (void)StripSuccess;
+  assert(StripSuccess && "Strings should be enclosed in quotes.");
+  return true;
+}
+
+// Describes a way to handle '\0' characters when processing the string.
+// rc.exe tool sometimes behaves in a weird way in postprocessing.
+// If the string to be output is equivalent to a C-string (e.g. in MENU
+// titles), string is (predictably) truncated after first 0-byte.
+// When outputting a string table, the behavior is equivalent to appending
+// '\0\0' at the end of the string, and then stripping the string
+// before the first '\0\0' occurrence.
+// Finally, when handling strings in user-defined resources, 0-bytes
+// aren't stripped, nor do they terminate the string.
+
+enum class NullHandlingMethod {
+  UserResource,   // Don't terminate string on '\0'.
+  CutAtNull,      // Terminate string on '\0'.
+  CutAtDoubleNull // Terminate string on '\0\0'; strip final '\0'.
+};
+
+// Parses an identifier or string and returns a processed version of it.
+// For now, it only strips the string boundaries, but TODO:
+//   * Squash "" to a single ".
+//   * Replace the escape sequences with their processed version.
+// For identifiers, this is no-op.
+static Error processString(StringRef Str, NullHandlingMethod NullHandler,
+                           bool &IsLongString, SmallVectorImpl<UTF16> &Result) {
+  assert(NullHandler == NullHandlingMethod::CutAtNull);
+
+  bool IsString = stripQuotes(Str, IsLongString);
+  convertUTF8ToUTF16String(Str, Result);
+
+  if (!IsString) {
+    // It's an identifier if it's not a string. Make all characters uppercase.
+    for (UTF16 &Ch : Result) {
+      assert(Ch <= 0x7F && "We didn't allow identifiers to be non-ASCII");
+      Ch = toupper(Ch);
+    }
+    return Error::success();
+  }
+
+  // We don't process the string contents. Only cut at '\0'.
+
+  for (size_t Pos = 0; Pos < Result.size(); ++Pos)
+    if (Result[Pos] == '\0')
+      Result.resize(Pos);
+
+  return Error::success();
+}
+
+uint64_t ResourceFileWriter::writeObject(const ArrayRef<uint8_t> Data) {
+  uint64_t Result = tell();
+  FS->write((const char *)Data.begin(), Data.size());
+  return Result;
+}
+
+Error ResourceFileWriter::writeCString(StringRef Str, bool WriteTerminator) {
+  SmallVector<UTF16, 128> ProcessedString;
+  bool IsLongString;
+  RETURN_IF_ERROR(processString(Str, NullHandlingMethod::CutAtNull,
+                                IsLongString, ProcessedString));
+  for (auto Ch : ProcessedString)
+    writeInt<uint16_t>(Ch);
+  if (WriteTerminator)
+    writeInt<uint16_t>(0);
+  return Error::success();
+}
+
+Error ResourceFileWriter::writeIdentifier(const IntOrString &Ident) {
+  return writeIntOrString(Ident);
+}
+
+Error ResourceFileWriter::writeIntOrString(const IntOrString &Value) {
+  if (!Value.isInt())
+    return writeCString(Value.getString());
+
+  writeInt<uint16_t>(0xFFFF);
+  writeInt<uint16_t>(Value.getInt());
+  return Error::success();
+}
+
+Error ResourceFileWriter::appendFile(StringRef Filename) {
+  bool IsLong;
+  stripQuotes(Filename, IsLong);
+
+  // Filename path should be relative to the current working directory.
+  // FIXME: docs say so, but reality is more complicated, script
+  // location and include paths must be taken into account.
+  ErrorOr<std::unique_ptr<MemoryBuffer>> File =
+      MemoryBuffer::getFile(Filename, -1, false);
+  if (!File)
+    return make_error<StringError>("Error opening file '" + Filename +
+                                       "': " + File.getError().message(),
+                                   File.getError());
+  *FS << (*File)->getBuffer();
+  return Error::success();
+}
+
+void ResourceFileWriter::padStream(uint64_t Length) {
+  assert(Length > 0);
+  uint64_t Location = tell();
+  Location %= Length;
+  uint64_t Pad = (Length - Location) % Length;
+  for (uint64_t i = 0; i < Pad; ++i)
+    writeInt<uint8_t>(0);
+}
+
+Error ResourceFileWriter::handleError(Error &&Err, const RCResource *Res) {
+  if (Err)
+    return joinErrors(createError("Error in " + Res->getResourceTypeName() +
+                                  " statement (ID " + Twine(Res->ResName) +
+                                  "): "),
+                      std::move(Err));
+  return Error::success();
+}
+
+Error ResourceFileWriter::visitNullResource(const RCResource *Res) {
+  return writeResource(Res, &ResourceFileWriter::writeNullBody);
+}
+
+Error ResourceFileWriter::visitHTMLResource(const RCResource *Res) {
+  return writeResource(Res, &ResourceFileWriter::writeHTMLBody);
+}
+
+Error ResourceFileWriter::visitLanguageStmt(const LanguageResource *Stmt) {
+  RETURN_IF_ERROR(checkNumberFits(Stmt->Lang, 10, "Primary language ID"));
+  RETURN_IF_ERROR(checkNumberFits(Stmt->SubLang, 6, "Sublanguage ID"));
+  ObjectData.LanguageInfo = Stmt->Lang | (Stmt->SubLang << 10);
+  return Error::success();
+}
+
+Error ResourceFileWriter::writeResource(
+    const RCResource *Res,
+    Error (ResourceFileWriter::*BodyWriter)(const RCResource *)) {
+  // We don't know the sizes yet.
+  object::WinResHeaderPrefix HeaderPrefix{ulittle32_t(0U), ulittle32_t(0U)};
+  uint64_t HeaderLoc = writeObject(HeaderPrefix);
+
+  auto ResType = Res->getResourceType();
+  RETURN_IF_ERROR(checkIntOrString(ResType, "Resource type"));
+  RETURN_IF_ERROR(checkIntOrString(Res->ResName, "Resource ID"));
+  RETURN_IF_ERROR(handleError(writeIdentifier(ResType), Res));
+  RETURN_IF_ERROR(handleError(writeIdentifier(Res->ResName), Res));
+
+  padStream(sizeof(uint32_t));
+  object::WinResHeaderSuffix HeaderSuffix{
+      ulittle32_t(0), // DataVersion; seems to always be 0
+      ulittle16_t(Res->getMemoryFlags()), ulittle16_t(ObjectData.LanguageInfo),
+      ulittle32_t(0),  // VersionInfo
+      ulittle32_t(0)}; // Characteristics
+  writeObject(HeaderSuffix);
+
+  uint64_t DataLoc = tell();
+  RETURN_IF_ERROR(handleError((this->*BodyWriter)(Res), Res));
+  // RETURN_IF_ERROR(handleError(dumpResource(Ctx)));
+
+  // Update the sizes.
+  HeaderPrefix.DataSize = tell() - DataLoc;
+  HeaderPrefix.HeaderSize = DataLoc - HeaderLoc;
+  writeObjectAt(HeaderPrefix, HeaderLoc);
+  padStream(sizeof(uint32_t));
+
+  return Error::success();
+}
+
+Error ResourceFileWriter::writeNullBody(const RCResource *) {
+  return Error::success();
+}
+
+Error ResourceFileWriter::writeHTMLBody(const RCResource *Base) {
+  return appendFile(cast<HTMLResource>(Base)->HTMLLoc);
+}
+
+} // namespace rc
+} // namespace llvm
diff --git a/tools/llvm-rc/ResourceFileWriter.h b/tools/llvm-rc/ResourceFileWriter.h
new file mode 100644
index 00000000000..2eaf2e377f4
--- /dev/null
+++ b/tools/llvm-rc/ResourceFileWriter.h
@@ -0,0 +1,89 @@
+//===-- ResourceSerializator.h ----------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This defines a visitor serializing resources to a .res stream.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVMRC_RESOURCESERIALIZATOR_H
+#define LLVM_TOOLS_LLVMRC_RESOURCESERIALIZATOR_H
+
+#include "ResourceScriptStmt.h"
+#include "ResourceVisitor.h"
+
+#include "llvm/Support/Endian.h"
+
+namespace llvm {
+namespace rc {
+
+class ResourceFileWriter : public Visitor {
+public:
+  ResourceFileWriter(std::unique_ptr<raw_fd_ostream> Stream)
+      : FS(std::move(Stream)) {
+    assert(FS && "Output stream needs to be provided to the serializator");
+  }
+
+  Error visitNullResource(const RCResource *) override;
+  Error visitHTMLResource(const RCResource *) override;
+
+  Error visitLanguageStmt(const LanguageResource *) override;
+
+  struct ObjectInfo {
+    uint16_t LanguageInfo;
+
+    ObjectInfo() : LanguageInfo(0) {}
+  } ObjectData;
+
+private:
+  Error handleError(Error &&Err, const RCResource *Res);
+
+  Error
+  writeResource(const RCResource *Res,
+                Error (ResourceFileWriter::*BodyWriter)(const RCResource *));
+
+  Error writeNullBody(const RCResource *);
+  Error writeHTMLBody(const RCResource *);
+
+  // Output stream handling.
+  std::unique_ptr<raw_fd_ostream> FS;
+
+  uint64_t tell() const { return FS->tell(); }
+
+  uint64_t writeObject(const ArrayRef<uint8_t> Data);
+
+  template <typename T> uint64_t writeInt(const T &Value) {
+    support::detail::packed_endian_specific_integral<T, support::little,
+                                                     support::unaligned>
+        Object(Value);
+    return writeObject(Object);
+  }
+
+  template <typename T> uint64_t writeObject(const T &Value) {
+    return writeObject(ArrayRef<uint8_t>(
+        reinterpret_cast<const uint8_t *>(&Value), sizeof(T)));
+  }
+
+  template <typename T> void writeObjectAt(const T &Value, uint64_t Position) {
+    FS->pwrite((const char *)&Value, sizeof(T), Position);
+  }
+
+  Error writeCString(StringRef Str, bool WriteTerminator = true);
+
+  Error writeIdentifier(const IntOrString &Ident);
+  Error writeIntOrString(const IntOrString &Data);
+
+  Error appendFile(StringRef Filename);
+
+  void padStream(uint64_t Length);
+};
+
+} // namespace rc
+} // namespace llvm
+
+#endif
diff --git a/tools/llvm-rc/ResourceScriptStmt.h b/tools/llvm-rc/ResourceScriptStmt.h
index 95890fe6669..362f765ae6e 100644
--- a/tools/llvm-rc/ResourceScriptStmt.h
+++ b/tools/llvm-rc/ResourceScriptStmt.h
@@ -15,6 +15,7 @@
 #define LLVM_TOOLS_LLVMRC_RESOURCESCRIPTSTMT_H
 
 #include "ResourceScriptToken.h"
+#include "ResourceVisitor.h"
 
 #include "llvm/ADT/StringSet.h"
 
@@ -49,15 +50,61 @@ public:
     return !IsInt && Data.String.equals_lower(Str);
   }
 
+  bool isInt() const { return IsInt; }
+
+  uint32_t getInt() const {
+    assert(IsInt);
+    return Data.Int;
+  }
+
+  const StringRef &getString() const {
+    assert(!IsInt);
+    return Data.String;
+  }
+
+  operator Twine() const {
+    return isInt() ? Twine(getInt()) : Twine(getString());
+  }
+
   friend raw_ostream &operator<<(raw_ostream &, const IntOrString &);
 };
 
+enum ResourceKind {
+  // These resource kinds have corresponding .res resource type IDs
+  // (TYPE in RESOURCEHEADER structure). The numeric value assigned to each
+  // kind is equal to this type ID.
+  RkNull = 0,
+  RkMenu = 4,
+  RkDialog = 5,
+  RkAccelerators = 9,
+  RkVersionInfo = 16,
+  RkHTML = 23,
+
+  // These kinds don't have assigned type IDs (they might be the resources
+  // of invalid kind, expand to many resource structures in .res files,
+  // or have variable type ID). In order to avoid ID clashes with IDs above,
+  // we assign the kinds the values 256 and larger.
+  RkInvalid = 256,
+  RkBase,
+  RkCursor,
+  RkIcon,
+  RkUser
+};
+
+// Non-zero memory flags.
+// Ref: msdn.microsoft.com/en-us/library/windows/desktop/ms648027(v=vs.85).aspx
+enum MemoryFlags {
+  MfMoveable = 0x10,
+  MfPure = 0x20,
+  MfPreload = 0x40,
+  MfDiscardable = 0x1000
+};
+
 // Base resource. All the resources should derive from this base.
 class RCResource {
-protected:
+public:
   IntOrString ResName;
 
-public:
   RCResource() = default;
   RCResource(RCResource &&) = default;
   void setName(const IntOrString &Name) { ResName = Name; }
@@ -65,6 +112,37 @@ public:
     return OS << "Base statement\n";
   };
   virtual ~RCResource() {}
+
+  virtual Error visit(Visitor *) const {
+    llvm_unreachable("This is unable to call methods from Visitor base");
+  }
+
+  // By default, memory flags are DISCARDABLE | PURE | MOVEABLE.
+  virtual uint16_t getMemoryFlags() const {
+    return MfDiscardable | MfPure | MfMoveable;
+  }
+  virtual ResourceKind getKind() const { return RkBase; }
+  static bool classof(const RCResource *Res) { return true; }
+
+  virtual IntOrString getResourceType() const {
+    llvm_unreachable("This cannot be called on objects without types.");
+  }
+  virtual Twine getResourceTypeName() const {
+    llvm_unreachable("This cannot be called on objects without types.");
+  };
+};
+
+// An empty resource. It has no content, type 0, ID 0 and all of its
+// characteristics are equal to 0.
+class NullResource : public RCResource {
+public:
+  raw_ostream &log(raw_ostream &OS) const override {
+    return OS << "Null resource\n";
+  }
+  Error visit(Visitor *V) const override { return V->visitNullResource(this); }
+  IntOrString getResourceType() const override { return 0; }
+  Twine getResourceTypeName() const override { return "(NULL)"; }
+  uint16_t getMemoryFlags() const override { return 0; }
 };
 
 // Optional statement base. All such statements should derive from this base.
@@ -89,12 +167,17 @@ public:
 //
 // Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa381019(v=vs.85).aspx
 class LanguageResource : public OptionalStmt {
+public:
   uint32_t Lang, SubLang;
 
-public:
   LanguageResource(uint32_t LangId, uint32_t SubLangId)
       : Lang(LangId), SubLang(SubLangId) {}
   raw_ostream &log(raw_ostream &) const override;
+
+  // This is not a regular top-level statement; when it occurs, it just
+  // modifies the language context.
+  Error visit(Visitor *V) const override { return V->visitLanguageStmt(this); }
+  Twine getResourceTypeName() const override { return "LANGUAGE"; }
 };
 
 // ACCELERATORS resource. Defines a named table of accelerators for the app.
@@ -161,11 +244,22 @@ public:
 //
 // Ref: msdn.microsoft.com/en-us/library/windows/desktop/aa966018(v=vs.85).aspx
 class HTMLResource : public RCResource {
+public:
   StringRef HTMLLoc;
 
-public:
   HTMLResource(StringRef Location) : HTMLLoc(Location) {}
   raw_ostream &log(raw_ostream &) const override;
+
+  Error visit(Visitor *V) const override { return V->visitHTMLResource(this); }
+
+  // Curiously, file resources don't have DISCARDABLE flag set.
+  uint16_t getMemoryFlags() const override { return MfPure | MfMoveable; }
+  IntOrString getResourceType() const override { return RkHTML; }
+  Twine getResourceTypeName() const override { return "HTML"; }
+  ResourceKind getKind() const override { return RkHTML; }
+  static bool classof(const RCResource *Res) {
+    return Res->getKind() == RkHTML;
+  }
 };
 
 // -- MENU resource and its helper classes --
diff --git a/tools/llvm-rc/ResourceVisitor.h b/tools/llvm-rc/ResourceVisitor.h
new file mode 100644
index 00000000000..9c6f3a78ee3
--- /dev/null
+++ b/tools/llvm-rc/ResourceVisitor.h
@@ -0,0 +1,39 @@
+//===-- ResourceVisitor.h ---------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This defines a base class visiting resource script resources.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVMRC_RESOURCEVISITOR_H
+#define LLVM_TOOLS_LLVMRC_RESOURCEVISITOR_H
+
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace rc {
+
+class RCResource;
+
+class LanguageResource;
+
+class Visitor {
+public:
+  virtual Error visitNullResource(const RCResource *) = 0;
+  virtual Error visitHTMLResource(const RCResource *) = 0;
+
+  virtual Error visitLanguageStmt(const LanguageResource *) = 0;
+
+  virtual ~Visitor() {}
+};
+
+} // namespace rc
+} // namespace llvm
+
+#endif
diff --git a/tools/llvm-rc/llvm-rc.cpp b/tools/llvm-rc/llvm-rc.cpp
index 9446b11a507..c62a037874b 100644
--- a/tools/llvm-rc/llvm-rc.cpp
+++ b/tools/llvm-rc/llvm-rc.cpp
@@ -12,12 +12,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "ResourceScriptToken.h"
+#include "ResourceFileWriter.h"
 #include "ResourceScriptParser.h"
+#include "ResourceScriptStmt.h"
+#include "ResourceScriptToken.h"
 
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Process.h"
@@ -27,6 +30,7 @@
 #include <system_error>
 
 using namespace llvm;
+using namespace llvm::rc;
 
 namespace {
 
@@ -134,11 +138,36 @@ int main(int argc_, const char *argv_[]) {
     }
   }
 
+  std::unique_ptr<ResourceFileWriter> Visitor;
+  bool IsDryRun = InputArgs.hasArg(OPT_DRY_RUN);
+
+  if (!IsDryRun) {
+    auto OutArgsInfo = InputArgs.getAllArgValues(OPT_FILEOUT);
+    if (OutArgsInfo.size() != 1)
+      fatalError(
+          "Exactly one output file should be provided (using /FO flag).");
+
+    std::error_code EC;
+    auto FOut =
+        llvm::make_unique<raw_fd_ostream>(OutArgsInfo[0], EC, sys::fs::F_RW);
+    if (EC)
+      fatalError("Error opening output file '" + OutArgsInfo[0] +
+                 "': " + EC.message());
+    Visitor = llvm::make_unique<ResourceFileWriter>(std::move(FOut));
+
+    ExitOnErr(NullResource().visit(Visitor.get()));
+
+    // Set the default language; choose en-US arbitrarily.
+    ExitOnErr(LanguageResource(0x09, 0x01).visit(Visitor.get()));
+  }
+
   rc::RCParser Parser{std::move(Tokens)};
   while (!Parser.isEof()) {
     auto Resource = ExitOnErr(Parser.parseSingleResource());
     if (BeVerbose)
       Resource->log(outs());
+    if (!IsDryRun)
+      ExitOnErr(Resource->visit(Visitor.get()));
   }
 
   return 0;