[linux-port] Enhance FS interface support (#1420)

There are various points where system functions are called via
OS-specific wrappers beginning with llvm::sys::fs. As part of the
development of the HLSL shader compiler, some of the microsoft
versions were altered to call into a further abstraction via the
MSFileSystem interface. In certain cases, this allows non-existent
"included" files to be accessed with the outward appearance of being
real files. However, many of these changes were missed by the Unix
abstraction. This adds them in with the facilitator of adding a few
functions to the MSFilesystem abstraction that better match Unix
system calls.

Needed for various tests that provide fake source files to compile
and include fake headers from within HLSL code. Otherwise, these
files are not found because they are not on the filesystem.

The Unix-like methods are added because in cases where the current
filesystem abstraction just calls the system calls directly, the
windows-style accesses were cumbersome and incomplete. Tablegen
and some tests make use of these simpler implementations.

include/dxc/Support/WinAdapter.h

@@ -107,16 +107,17 @@
 
 // Map these errors to equivalent errnos.
 #define ERROR_SUCCESS 0L
-#define ERROR_OUT_OF_STRUCTURES ENOMEM
-#define ERROR_UNHANDLED_EXCEPTION EINTR
-#define ERROR_NOT_FOUND ENOTSUP
-#define ERROR_NOT_CAPABLE EPERM
+#define ERROR_ARITHMETIC_OVERFLOW EOVERFLOW
 #define ERROR_FILE_NOT_FOUND ENOENT
+#define ERROR_FUNCTION_NOT_CALLED ENOSYS
 #define ERROR_IO_DEVICE EIO
-#define ERROR_INVALID_HANDLE EBADF
-#define ERROR_ARITHMETIC_OVERFLOW EOVERFLOW
 #define ERROR_INSUFFICIENT_BUFFER ENOBUFS
+#define ERROR_INVALID_HANDLE EBADF
 #define ERROR_INVALID_PARAMETER EINVAL
+#define ERROR_OUT_OF_STRUCTURES ENOMEM
+#define ERROR_NOT_CAPABLE EPERM
+#define ERROR_NOT_FOUND ENOTSUP
+#define ERROR_UNHANDLED_EXCEPTION EINTR
 
 // Used by HRESULT <--> WIN32 error code conversion
 #define SEVERITY_ERROR 1

include/llvm/Support/MSFileSystem.h

@@ -14,6 +14,7 @@
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // MSFileSystem interface.
+struct stat;
 
 namespace llvm {
 namespace sys  {
@@ -86,6 +87,13 @@ public:
   virtual errno_t resize_file(_In_ LPCWSTR path, uint64_t size) throw() = 0; // A number of C calls.
   virtual int Read(int fd, _Out_bytecap_(count) void* buffer, unsigned int count) throw() = 0;
   virtual int Write(int fd, _In_bytecount_(count) const void* buffer, unsigned int count) throw() = 0;
+
+  // Unix interface
+#ifndef _WIN32
+  virtual int Open(const char *lpFileName, int flags, mode_t mode = 0) throw() = 0;
+  virtual int Stat(const char *lpFileName, struct stat *Status) throw() = 0;
+  virtual int Fstat(int FD, struct stat *Status) throw() = 0;
+#endif
 };
 
 

lib/MSSupport/MSFileSystemImpl.cpp

@@ -89,6 +89,11 @@ public:
   virtual errno_t resize_file(_In_ LPCWSTR path, uint64_t size) throw() override;
   virtual int Read(int fd, _Out_bytecap_(count) void* buffer, unsigned int count) throw() override;
   virtual int Write(int fd, _In_bytecount_(count) const void* buffer, unsigned int count) throw() override;
+#ifndef _WIN32
+  virtual int Open(const char *lpFileName, int flags, mode_t mode) throw() override;
+  virtual int Stat(const char *lpFileName, struct stat *Status) throw() override;
+  virtual int Fstat(int FD, struct stat *Status) throw() override;
+#endif
 };
 
 MSFileSystemForDisk::MSFileSystemForDisk()
@@ -516,6 +521,20 @@ int MSFileSystemForDisk::Write(int fd, const void* buffer, unsigned int count) t
   #endif
 }
 
+#ifndef _WIN32
+int MSFileSystemForDisk::Open(const char *lpFileName, int flags, mode_t mode) throw() {
+  return ::open(lpFileName, flags, mode);
+}
+
+int MSFileSystemForDisk::Stat(const char *lpFileName, struct stat *Status) throw() {
+  return ::stat(lpFileName, Status);
+}
+
+int MSFileSystemForDisk::Fstat(int FD, struct stat *Status) throw() {
+  return ::fstat(FD, Status);
+}
+#endif
+
 } // end namespace fs
 } // end namespace sys
 } // end namespace llvm

lib/Support/MSFileSystemBasic.cpp

@@ -222,6 +222,11 @@ public:
   virtual errno_t resize_file(_In_ LPCWSTR path, uint64_t size) throw() override;
   virtual int Read(int fd, _Out_bytecap_(count) void* buffer, unsigned int count) throw() override;
   virtual int Write(int fd, _In_bytecount_(count) const void* buffer, unsigned int count) throw() override;
+#ifndef _WIN32
+  virtual int Open(const char *lpFileName, int flags, mode_t mode) throw() override;
+  virtual int Stat(const char *lpFileName, struct stat *Status) throw() override;
+  virtual int Fstat(int FD, struct stat *Status) throw() override;
+#endif
 };
 
 _Use_decl_annotations_
@@ -942,6 +947,23 @@ Cleanup:
   return (int)cbWritten;
 }
 
+#ifndef _WIN32
+int MSFileSystemForIface::Open(const char *lpFileName, int flags, mode_t mode) throw() {
+  SetLastError(ERROR_FUNCTION_NOT_CALLED);
+  return FALSE;
+}
+
+int MSFileSystemForIface::Stat(const char *lpFileName, struct stat *Status) throw() {
+  SetLastError(ERROR_FUNCTION_NOT_CALLED);
+  return FALSE;
+}
+
+int MSFileSystemForIface::Fstat(int FD, struct stat *Status) throw() {
+  SetLastError(ERROR_FUNCTION_NOT_CALLED);
+  return FALSE;
+}
+#endif
+
 } // end namespace fs
 } // end namespace sys
 } // end namespace llvm
@@ -1099,6 +1121,18 @@ public:
   virtual int Write(int fd, const void* buffer, unsigned int count) throw() override
   { return MSFileSystemBlockedErrno(); }
 
+  // Unix interface
+#ifndef _WIN32
+  virtual int Open(const char *lpFileName, int flags, mode_t mode) throw() override
+  { return MSFileSystemBlockedErrno(); }
+
+  virtual int Stat(const char *lpFileName, struct stat *Status) throw() override
+  { return MSFileSystemBlockedErrno(); }
+
+  virtual int Fstat(int FD, struct stat *Status) throw() override
+  { return MSFileSystemBlockedErrno(); }
+#endif
+
 };
 
 MSFileSystemBlocked::MSFileSystemBlocked()

lib/Support/MemoryBuffer.cpp

@@ -377,6 +377,7 @@ getOpenFileImpl(int FD, const Twine &Filename, uint64_t FileSize,
   char *BufPtr = const_cast<char *>(Buf->getBufferStart());
 
   size_t BytesLeft = MapSize;
+#undef HAVE_PREAD // HLSL Change - pread bypasses needed layers
 #ifndef HAVE_PREAD
   if (llvm::sys::fs::msf_lseek(FD, Offset, SEEK_SET) == -1)  // HLSL Change - use msf_lseek
     return std::error_code(errno, std::generic_category());

lib/Support/Unix/Path.inc

@@ -60,6 +60,8 @@
 # define PATH_MAX 4096
 #endif
 
+#include "llvm/Support/MSFileSystem.h" // HLSL change
+
 using namespace llvm;
 
 namespace llvm {
@@ -379,13 +381,15 @@ std::error_code status(const Twine &Path, file_status &Result) {
   StringRef P = Path.toNullTerminatedStringRef(PathStorage);
 
   struct stat Status;
-  int StatRet = ::stat(P.begin(), &Status);
+  MSFileSystem* fsr = GetCurrentThreadFileSystem(); // HLSL Change
+  int StatRet = fsr->Stat(P.begin(), &Status); // HLSL Change - FS abstraction
   return fillStatus(StatRet, Status, Result);
 }
 
 std::error_code status(int FD, file_status &Result) {
   struct stat Status;
-  int StatRet = ::fstat(FD, &Status);
+  MSFileSystem* fsr = GetCurrentThreadFileSystem(); // HLSL Change
+  int StatRet = fsr->Fstat(FD, &Status); // HLSL Change - FS abstraction
   return fillStatus(StatRet, Status, Result);
 }
 
@@ -504,7 +508,8 @@ std::error_code detail::directory_iterator_increment(detail::DirIterState &it) {
 std::error_code openFileForRead(const Twine &Name, int &ResultFD) {
   SmallString<128> Storage;
   StringRef P = Name.toNullTerminatedStringRef(Storage);
-  while ((ResultFD = open(P.begin(), O_RDONLY)) < 0) {
+  MSFileSystem* fsr = GetCurrentThreadFileSystem(); // HLSL Change
+  while ((ResultFD = fsr->Open(P.begin(), O_RDONLY)) < 0) { // HLSL Change - FS abstraction
     if (errno != EINTR)
       return std::error_code(errno, std::generic_category());
   }
@@ -534,7 +539,8 @@ std::error_code openFileForWrite(const Twine &Name, int &ResultFD,
 
   SmallString<128> Storage;
   StringRef P = Name.toNullTerminatedStringRef(Storage);
-  while ((ResultFD = open(P.begin(), OpenFlags, Mode)) < 0) {
+  MSFileSystem* fsr = GetCurrentThreadFileSystem(); // HLSL Change
+  while ((ResultFD = fsr->Open(P.begin(), OpenFlags, Mode)) < 0) { // HLSL Change - FS abstraction
     if (errno != EINTR)
       return std::error_code(errno, std::generic_category());
   }
@@ -624,7 +630,6 @@ void system_temp_directory(bool ErasedOnReboot, SmallVectorImpl<char> &Result) {
 // The following is implementation of MSFileSystem for Unix.
 // This will make usages of MSFileSystem throughout DirectXShaderCompiler
 // compatible with Unix.
-#include "llvm/Support/MSFileSystem.h"
 namespace llvm {
 namespace sys  {
 namespace fs {
@@ -700,6 +705,6 @@ int msf_setmode(int fd, int mode) throw() {
   return fsr->setmode(fd, mode);
 }
 
-}
-}
-}
+} // namespace fs
+} // namespace sys
+} // namespace llvm

lib/Support/Unix/Process.inc

@@ -262,7 +262,7 @@ std::error_code Process::SafelyCloseFileDescriptor(int FD) {
   // We need to save the error, if one occurs, because our subsequent call to
   // pthread_sigmask might tamper with errno.
   int ErrnoFromClose = 0;
-  if (::close(FD) < 0)
+  if (fs::msf_close(FD) < 0)
     ErrnoFromClose = errno;
   // Restore the signal mask back to what we saved earlier.
   int EC = 0;

tools/clang/tools/dxcompiler/dxcfilesystem.cpp

@@ -21,6 +21,11 @@
 #include "dxc/Support/Unicode.h"
 #include "clang/Frontend/CompilerInstance.h"
 
+#ifndef _WIN32
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
+
 using namespace llvm;
 using namespace hlsl;
 
@@ -732,6 +737,78 @@ public:
 
     return (int)written;
   }
+#ifndef _WIN32
+  // Replicate Unix interfaces using DxcArgsFileSystemImpl
+
+  int getStatus(HANDLE FileHandle, struct stat *Status) {
+    if (FileHandle == INVALID_HANDLE_VALUE)
+      return -1;
+
+    memset(Status, 0, sizeof(*Status));
+    switch (GetFileType(FileHandle)) {
+      default:
+        llvm_unreachable("Don't know anything about this file type");
+      case FILE_TYPE_DISK:
+        break;
+      case FILE_TYPE_CHAR:
+        Status->st_mode = S_IFCHR;
+        return 0;
+      case FILE_TYPE_PIPE:
+        Status->st_mode = S_IFIFO;
+        return 0;
+      }
+
+    BY_HANDLE_FILE_INFORMATION Info;
+    if (!GetFileInformationByHandle(FileHandle, &Info))
+      return -1;
+
+    if (Info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+      Status->st_mode = S_IFDIR;
+    else
+      Status->st_mode = S_IFREG;
+
+    Status->st_dev = Info.nFileIndexHigh;
+    Status->st_ino = Info.nFileIndexLow;
+    Status->st_mtime = Info.ftLastWriteTime.dwLowDateTime;
+    Status->st_size = Info.nFileSizeLow;
+    return 0;
+  }
+
+  virtual int Open(const char *lpFileName, int flags, mode_t mode) throw() override {
+    HANDLE H = CreateFileW(CA2W(lpFileName), GENERIC_READ | GENERIC_WRITE,
+                           FILE_SHARE_READ | FILE_SHARE_WRITE,
+                           OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL);
+    if (H == INVALID_HANDLE_VALUE)
+      return -1;
+    int FD = open_osfhandle(intptr_t(H), 0);
+    if (FD == -1)
+      CloseHandle(H);
+    return FD;
+  }
+
+  // fake my way toward as linux-y a file_status as I can get
+  virtual int Stat(const char *lpFileName, struct stat *Status) throw() override {
+    CA2W fileName_utf16(lpFileName);
+
+    DWORD attr = GetFileAttributesW(fileName_utf16);
+    if (attr == INVALID_FILE_ATTRIBUTES)
+      return -1;
+
+    HANDLE H = CreateFileW(fileName_utf16, 0, // Attributes only.
+                           FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE,
+                           OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL);
+    if (H == INVALID_HANDLE_VALUE)
+      return -1;
+
+    return getStatus(H, Status);
+  }
+
+  virtual int Fstat(int FD, struct stat *Status) throw() override {
+    HANDLE FileHandle = reinterpret_cast<HANDLE>(get_osfhandle(FD));
+    return getStatus(FileHandle, Status);
+  }
+#endif // _WIN32
+
 };
 }