Index whild build. Part 1

https://reviews.llvm.org/D39050 adapted for 6.0

Change-Id: I2cb9c7d94c95fde7c3eb3042a6de0e55ab186974
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>

54 files changed, 1827 insertions, 140 deletions

diff --git a/include/clang/Basic/AllDiagnostics.h b/include/clang/Basic/AllDiagnostics.h
index 1c83e2d0f8..5c61996024 100644
--- a/include/clang/Basic/AllDiagnostics.h
+++ b/include/clang/Basic/AllDiagnostics.h
@@ -21,6 +21,7 @@
 #include "clang/CrossTU/CrossTUDiagnostic.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Index/IndexDiagnostic.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Sema/SemaDiagnostic.h"
diff --git a/include/clang/Basic/CMakeLists.txt b/include/clang/Basic/CMakeLists.txt
index 821c405913..b2dd03630e 100644
--- a/include/clang/Basic/CMakeLists.txt
+++ b/include/clang/Basic/CMakeLists.txt
@@ -12,6 +12,7 @@ clang_diag_gen(Common)
 clang_diag_gen(CrossTU)
 clang_diag_gen(Driver)
 clang_diag_gen(Frontend)
+clang_diag_gen(Index)
 clang_diag_gen(Lex)
 clang_diag_gen(Parse)
 clang_diag_gen(Refactoring)
diff --git a/include/clang/Basic/Diagnostic.td b/include/clang/Basic/Diagnostic.td
index 52ccf350e6..3cc357c434 100644
--- a/include/clang/Basic/Diagnostic.td
+++ b/include/clang/Basic/Diagnostic.td
@@ -136,6 +136,7 @@ include "DiagnosticCommonKinds.td"
 include "DiagnosticCrossTUKinds.td"
 include "DiagnosticDriverKinds.td"
 include "DiagnosticFrontendKinds.td"
+include "DiagnosticIndexKinds.td"
 include "DiagnosticLexKinds.td"
 include "DiagnosticParseKinds.td"
 include "DiagnosticRefactoringKinds.td"
diff --git a/include/clang/Basic/DiagnosticGroups.td b/include/clang/Basic/DiagnosticGroups.td
index 2d471f1fa5..40b5aef08c 100644
--- a/include/clang/Basic/DiagnosticGroups.td
+++ b/include/clang/Basic/DiagnosticGroups.td
@@ -327,6 +327,7 @@ def MissingFieldInitializers : DiagGroup<"missing-field-initializers">;
 def ModuleBuild : DiagGroup<"module-build">;
 def ModuleConflict : DiagGroup<"module-conflict">;
 def ModuleFileExtension : DiagGroup<"module-file-extension">;
+def IndexStore : DiagGroup<"index-store">;
 def NewlineEOF : DiagGroup<"newline-eof">;
 def Nullability : DiagGroup<"nullability">;
 def NullabilityDeclSpec : DiagGroup<"nullability-declspec">;
diff --git a/include/clang/Basic/DiagnosticIDs.h b/include/clang/Basic/DiagnosticIDs.h
index b4ea85ba85..92792097c8 100644
--- a/include/clang/Basic/DiagnosticIDs.h
+++ b/include/clang/Basic/DiagnosticIDs.h
@@ -40,6 +40,7 @@ namespace clang {
       DIAG_SIZE_SEMA          = 3500,
       DIAG_SIZE_ANALYSIS      =  100,
       DIAG_SIZE_REFACTORING   = 1000,
+      DIAG_SIZE_INDEX         =  100,
     };
     // Start position for diagnostics.
     enum {
@@ -55,7 +56,8 @@ namespace clang {
       DIAG_START_SEMA          = DIAG_START_CROSSTU       + DIAG_SIZE_COMMENT,
       DIAG_START_ANALYSIS      = DIAG_START_SEMA          + DIAG_SIZE_SEMA,
       DIAG_START_REFACTORING   = DIAG_START_ANALYSIS      + DIAG_SIZE_ANALYSIS,
-      DIAG_UPPER_LIMIT         = DIAG_START_REFACTORING   + DIAG_SIZE_REFACTORING
+      DIAG_START_INDEX         = DIAG_START_REFACTORING   + DIAG_SIZE_REFACTORING,
+      DIAG_UPPER_LIMIT         = DIAG_START_INDEX         + DIAG_SIZE_INDEX,
     };
 
     class CustomDiagInfo;
diff --git a/include/clang/Basic/DiagnosticIndexKinds.td b/include/clang/Basic/DiagnosticIndexKinds.td
new file mode 100644
index 0000000000..0f0525847a
--- /dev/null
+++ b/include/clang/Basic/DiagnosticIndexKinds.td
@@ -0,0 +1,31 @@
+//==--- DiagnosticIndexKinds.td - indexing diagnostics --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Indexing Diagnostics
+//===----------------------------------------------------------------------===//
+
+let Component = "Index" in {
+
+let CategoryName = "Index Store Issue" in {
+
+def err_index_store_dir_create_failed : Error<"failed creating the index store "
+  "directory: %0">;
+def err_index_store_file_status_failed : Error<"failed file status check: %0">;
+def err_index_store_record_write_failed : Error<"failed writing record '%0': "
+  "%1">;
+def err_index_store_unit_write_failed : Error<"failed writing unit data: %0">;
+
+def remark_index_producing_module_file_data : Remark<"producing index data for "
+  "module file '%0'">,
+  InGroup<IndexStore>;
+
+}
+
+} // end of Indexing diagnostics
diff --git a/include/clang/Driver/Job.h b/include/clang/Driver/Job.h
index b74b3b4b35..b45ae92f26 100644
--- a/include/clang/Driver/Job.h
+++ b/include/clang/Driver/Job.h
@@ -34,9 +34,11 @@ using llvm::opt::ArgStringList;
 struct CrashReportInfo {
   StringRef Filename;
   StringRef VFSPath;
+  StringRef IndexStorePath;
 
-  CrashReportInfo(StringRef Filename, StringRef VFSPath)
-      : Filename(Filename), VFSPath(VFSPath) {}
+  CrashReportInfo(StringRef Filename, StringRef VFSPath,
+                  StringRef IndexStorePath)
+      : Filename(Filename), VFSPath(VFSPath), IndexStorePath(IndexStorePath) {}
 };
 
 /// Command - An executable path/name and argument vector to
diff --git a/include/clang/Driver/Options.td b/include/clang/Driver/Options.td
index 288bd482bb..504bb0ddf7 100644
--- a/include/clang/Driver/Options.td
+++ b/include/clang/Driver/Options.td
@@ -324,6 +324,13 @@ def objcmt_whitelist_dir_path: Joined<["-"], "objcmt-whitelist-dir-path=">, Flag
 def : Joined<["-"], "objcmt-white-list-dir-path=">, Flags<[CC1Option]>,
     Alias<objcmt_whitelist_dir_path>;
 
+def index_store_path : Separate<["-"], "index-store-path">, Flags<[CC1Option]>,
+  HelpText<"Enable indexing with the specified data store path">;
+def index_ignore_system_symbols : Flag<["-"], "index-ignore-system-symbols">, Flags<[CC1Option]>,
+  HelpText<"Ignore symbols from system headers">;
+def index_record_codegen_name : Flag<["-"], "index-record-codegen-name">, Flags<[CC1Option]>,
+  HelpText<"Record the codegen name for symbols">;
+
 // Make sure all other -ccc- options are rejected.
 def ccc_ : Joined<["-"], "ccc-">, Group<internal_Group>, Flags<[Unsupported]>;
 
diff --git a/include/clang/Frontend/CompilerInstance.h b/include/clang/Frontend/CompilerInstance.h
index 90a9501475..148c639e03 100644
--- a/include/clang/Frontend/CompilerInstance.h
+++ b/include/clang/Frontend/CompilerInstance.h
@@ -183,6 +183,14 @@ class CompilerInstance : public ModuleLoader {
   /// The list of active output files.
   std::list<OutputFile> OutputFiles;
 
+  typedef std::function<std::unique_ptr<FrontendAction>(
+      const FrontendOptions &Opts, std::unique_ptr<FrontendAction> Action)>
+      ActionWrapperTy;
+
+  /// \brief An optional callback function used to wrap any
+  /// GenerateModuleActions created and executed when loading modules.
+  ActionWrapperTy GenModuleActionWrapper;
+
   CompilerInstance(const CompilerInstance &) = delete;
   void operator=(const CompilerInstance &) = delete;
 public:
@@ -447,7 +455,7 @@ public:
     return *PP;
   }
 
-  std::shared_ptr<Preprocessor> getPreprocessorPtr() { return PP; }
+  std::shared_ptr<Preprocessor> getPreprocessorPtr() const { return PP; }
 
   void resetAndLeakPreprocessor() {
     BuryPointer(new std::shared_ptr<Preprocessor>(PP));
@@ -796,6 +804,14 @@ public:
 
   bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override;
 
+  void setGenModuleActionWrapper(ActionWrapperTy Wrapper) {
+    GenModuleActionWrapper = Wrapper;
+  };
+
+  ActionWrapperTy getGenModuleActionWrapper() const {
+    return GenModuleActionWrapper;
+  }
+
   void addDependencyCollector(std::shared_ptr<DependencyCollector> Listener) {
     DependencyCollectors.push_back(std::move(Listener));
   }
diff --git a/include/clang/Frontend/FrontendOptions.h b/include/clang/Frontend/FrontendOptions.h
index 5192a3774c..72fa3b6eac 100644
--- a/include/clang/Frontend/FrontendOptions.h
+++ b/include/clang/Frontend/FrontendOptions.h
@@ -259,6 +259,13 @@ public:
   std::string MTMigrateDir;
   std::string ARCMTMigrateReportOut;
 
+  /// The path to write index data to
+  std::string IndexStorePath;
+  /// Whether to ignore system files when writing out index data
+  unsigned IndexIgnoreSystemSymbols : 1;
+  /// Whether to include the codegen name of symbols in the index data
+  unsigned IndexRecordCodegenName : 1;
+
   /// The input files and their types.
   std::vector<FrontendInputFile> Inputs;
 
@@ -337,7 +344,8 @@ public:
     GenerateGlobalModuleIndex(true), ASTDumpDecls(false), ASTDumpLookups(false),
     BuildingImplicitModule(false), ModulesEmbedAllFiles(false),
     IncludeTimestamps(true), ARCMTAction(ARCMT_None),
-    ObjCMTAction(ObjCMT_None), ProgramAction(frontend::ParseSyntaxOnly)
+    ObjCMTAction(ObjCMT_None), ProgramAction(frontend::ParseSyntaxOnly),
+    IndexIgnoreSystemSymbols(false), IndexRecordCodegenName(false)
   {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
diff --git a/include/clang/Index/DeclOccurrence.h b/include/clang/Index/DeclOccurrence.h
new file mode 100644
index 0000000000..44d7f85620
--- /dev/null
+++ b/include/clang/Index/DeclOccurrence.h
@@ -0,0 +1,42 @@
+//===--- DeclOccurrence.h - An occurrence of a decl within a file ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INDEX_DECLOCCURRENCE_H
+#define LLVM_CLANG_INDEX_DECLOCCURRENCE_H
+
+#include "clang/Basic/LLVM.h"
+#include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace clang {
+class Decl;
+
+namespace index {
+
+struct DeclOccurrence {
+  SymbolRoleSet Roles;
+  unsigned Offset;
+  const Decl *Dcl;
+  SmallVector<SymbolRelation, 3> Relations;
+
+  DeclOccurrence(SymbolRoleSet R, unsigned Offset, const Decl *D,
+                 ArrayRef<SymbolRelation> Relations)
+      : Roles(R), Offset(Offset), Dcl(D),
+        Relations(Relations.begin(), Relations.end()) {}
+
+  friend bool operator<(const DeclOccurrence &LHS, const DeclOccurrence &RHS) {
+    return LHS.Offset < RHS.Offset;
+  }
+};
+
+} // namespace index
+} // namespace clang
+
+#endif
diff --git a/include/clang/Index/IndexDataConsumer.h b/include/clang/Index/IndexDataConsumer.h
index 080f4cb4d0..7cfaa15132 100644
--- a/include/clang/Index/IndexDataConsumer.h
+++ b/include/clang/Index/IndexDataConsumer.h
@@ -43,17 +43,18 @@ public:
   virtual bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles,
                                    ArrayRef<SymbolRelation> Relations,
                                    FileID FID, unsigned Offset,
+                                   bool IsInSystemFile,
                                    ASTNodeInfo ASTNode);
 
   /// \returns true to continue indexing, or false to abort.
   virtual bool handleMacroOccurence(const IdentifierInfo *Name,
                                     const MacroInfo *MI, SymbolRoleSet Roles,
-                                    FileID FID, unsigned Offset);
+                                    FileID FID, unsigned Offset, bool IsInSystemFile);
 
   /// \returns true to continue indexing, or false to abort.
   virtual bool handleModuleOccurence(const ImportDecl *ImportD,
                                      SymbolRoleSet Roles,
-                                     FileID FID, unsigned Offset);
+                                     FileID FID, unsigned Offset, bool IsInSystemFile);
 
   virtual void finish() {}
 
diff --git a/include/clang/Index/IndexDiagnostic.h b/include/clang/Index/IndexDiagnostic.h
new file mode 100644
index 0000000000..b2903b9df7
--- /dev/null
+++ b/include/clang/Index/IndexDiagnostic.h
@@ -0,0 +1,29 @@
+//===--- IndexDiagnostic.h - ------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INDEX_INDEXDIAGNOSTIC_H
+#define LLVM_CLANG_INDEX_INDEXDIAGNOSTIC_H
+
+#include "clang/Basic/Diagnostic.h"
+
+namespace clang {
+namespace diag {
+enum {
+#define DIAG(ENUM, FLAGS, DEFAULT_MAPPING, DESC, GROUP, SFINAE, NOWERROR,      \
+             SHOWINSYSHEADER, CATEGORY)                                        \
+  ENUM,
+#define INDEXSTART
+#include "clang/Basic/DiagnosticIndexKinds.inc"
+#undef DIAG
+  NUM_BUILTIN_INDEX_DIAGNOSTICS
+};
+} // end namespace diag
+} // end namespace clang
+
+#endif // LLVM_CLANG_INDEX_INDEXDIAGNOSTIC_H
diff --git a/include/clang/Index/IndexingAction.h b/include/clang/Index/IndexingAction.h
index fb703be4e5..5b0585695a 100644
--- a/include/clang/Index/IndexingAction.h
+++ b/include/clang/Index/IndexingAction.h
@@ -1,4 +1,4 @@
-//===--- IndexingAction.h - Frontend index action -------------------------===//
+//===--- IndexingAction.h - Frontend AST indexing action ------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -13,6 +13,7 @@
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/ArrayRef.h"
 #include <memory>
+#include <string>
 
 namespace clang {
   class ASTContext;
@@ -35,11 +36,14 @@ struct IndexingOptions {
     All,
   };
 
-  SystemSymbolFilterKind SystemSymbolFilter
-    = SystemSymbolFilterKind::DeclarationsOnly;
+  SystemSymbolFilterKind SystemSymbolFilter =
+      SystemSymbolFilterKind::DeclarationsOnly;
   bool IndexFunctionLocals = false;
 };
 
+/// Creates a frontend action that provides decl occurrence information from the
+/// AST to the given \c IndexDataConsumer.
+///
 /// \param WrappedAction another frontend action to wrap over or null.
 std::unique_ptr<FrontendAction>
 createIndexingAction(std::shared_ptr<IndexDataConsumer> DataConsumer,
diff --git a/include/clang/Index/RecordingAction.h b/include/clang/Index/RecordingAction.h
new file mode 100644
index 0000000000..15e987574c
--- /dev/null
+++ b/include/clang/Index/RecordingAction.h
@@ -0,0 +1,59 @@
+//===--- RecordingAction.h - Frontend index recording action --------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INDEX_INDEXRECORDINGACTION_H
+#define LLVM_CLANG_INDEX_INDEXRECORDINGACTION_H
+
+#include "clang/Basic/LLVM.h"
+#include "clang/Index/UnitIndexingAction.h"
+#include "llvm/ADT/ArrayRef.h"
+
+namespace clang {
+class CompilerInstance;
+class FrontendAction;
+class FrontendOptions;
+
+namespace serialization {
+class ModuleFile;
+}
+
+namespace index {
+
+struct RecordingOptions : UnitIndexingOptions {
+  std::string DataDirPath;
+  bool RecordSymbolCodeGenName = false;
+};
+
+RecordingOptions
+getRecordingOptionsFromFrontendOptions(const FrontendOptions &FEOpts);
+
+/// \brief Creates a frontend action that collects dependency, file inclusion
+/// and decl ocurrence information for the translation unit and persists it to
+/// an index store.
+///
+/// FIXME: Not implemented yet.
+///
+/// \param WrappedAction another frontend action to wrap over or null.
+std::unique_ptr<FrontendAction>
+createIndexDataRecordingAction(RecordingOptions RecordOpts,
+                               std::unique_ptr<FrontendAction> WrappedAction);
+
+/// Collects dependency, file inclusion and decl occurrence information for a
+/// \c ModuleFile and persists it to an index store. Does \b not check if
+/// the store already has up-to-date information for the provided module file.
+///
+/// FIXME: Not implemented yet.
+void recordIndexDataForModuleFile(serialization::ModuleFile *ModFile,
+                                  RecordingOptions RecordOpts,
+                                  const CompilerInstance &CI);
+
+} // namespace index
+} // namespace clang
+
+#endif
diff --git a/include/clang/Index/UnitIndexDataConsumer.h b/include/clang/Index/UnitIndexDataConsumer.h
new file mode 100644
index 0000000000..9ee0d82b9a
--- /dev/null
+++ b/include/clang/Index/UnitIndexDataConsumer.h
@@ -0,0 +1,74 @@
+//===--- UnitIndexDataConsumer.h - Abstract unit index data consumer ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INDEX_UNITINDEXDATACONSUMER_H
+#define LLVM_CLANG_INDEX_UNITINDEXDATACONSUMER_H
+
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Index/DeclOccurrence.h"
+#include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/ArrayRef.h"
+
+namespace clang {
+namespace serialization {
+class ModuleFile;
+}
+
+namespace index {
+
+/// Consumer for the index data associated with a translation unit.
+class UnitIndexDataConsumer {
+public:
+  virtual ~UnitIndexDataConsumer() = default;
+
+  /// Called for each file dependency of the translation unit.
+  virtual void handleFileDependency(const FileEntry *FE, bool IsSystem) {}
+
+  /// Called for each file include in the translation unit.
+  virtual void handleInclude(const FileEntry *Source, unsigned Line,
+                             const FileEntry *Target) {}
+
+  /// Called for each each module imported by the translation unit.
+  virtual void handleModuleImport(const serialization::ModuleFile &Mod,
+                                  bool IsSystem) {}
+
+  /// Determines whether to collect the index data associated with the given
+  /// dependency of this translation unit or not.
+  ///
+  /// \param OutFilePath the output file path of the dependency.
+  /// \returns true to collect index data for \c Mod.
+  virtual bool
+  shouldIndexModuleDependency(const serialization::ModuleFile &Mod) {
+    return false;
+  }
+
+  /// Called with the decl occurrences in each file and AST file dependency,
+  /// sorted by offset.
+  ///
+  /// \returns true to cancel consuming data for this translation unit. Finish
+  /// will not be called.
+  virtual bool
+  handleFileOccurrences(FileID FID,
+                        ArrayRef<DeclOccurrence> OccurrencesSortedByOffset,
+                        bool IsSystem) {
+    return false;
+  }
+
+  /// Called when there is no more data to handle.
+  virtual void finish() {}
+
+private:
+  // avoid duplicate vtables
+  virtual void _anchor();
+};
+
+} // namespace index
+} // namespace clang
+
+#endif
diff --git a/include/clang/Index/UnitIndexingAction.h b/include/clang/Index/UnitIndexingAction.h
new file mode 100644
index 0000000000..f661093941
--- /dev/null
+++ b/include/clang/Index/UnitIndexingAction.h
@@ -0,0 +1,87 @@
+//===--- UnitIndexingAction.h - Frontend unit indexing action -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INDEX_UNITINDEXINGACTION_H
+#define LLVM_CLANG_INDEX_UNITINDEXINGACTION_H
+
+#include "clang/Basic/LLVM.h"
+#include "clang/Index/IndexingAction.h"
+#include "llvm/ADT/ArrayRef.h"
+
+namespace clang {
+class CompilerInstance;
+class FileEntry;
+class FrontendAction;
+class Module;
+
+namespace serialization {
+class ModuleFile;
+}
+
+namespace index {
+class UnitIndexDataConsumer;
+
+struct UnitIndexingOptions : IndexingOptions {
+  enum class FileIncludeFilterKind {
+    None,
+    UserOnly, // only record includes inside non-system files.
+    All,
+  };
+
+  bool IncludeSystemDependencies = true;
+  FileIncludeFilterKind FileIncludeFilter = FileIncludeFilterKind::UserOnly;
+};
+
+/// \brief Information about a translation unit useful for indexing.
+///
+struct UnitDetails {
+  const CompilerInstance &CI; ///< The owning compiler instance.
+
+  Module *UnitModule;     ///< The corresponding \c Module (module units only).
+  std::string ModuleName; ///< The \c Module name (module units only).
+  const FileEntry *RootFile; ///< The root \c FileEntry (non-module units only).
+
+  std::string OutputFile; ///< The output file path.
+  StringRef SysrootPath;  ///< The "virtual system root" path.
+  bool IsSystemUnit;
+  bool IsModuleUnit;
+  bool IsDebugCompilation;
+};
+
+/// Factory function type for producing UnitIndexDataConsumers for a given
+/// translation unit
+typedef std::function<std::unique_ptr<UnitIndexDataConsumer>(
+    UnitDetails UnitInfo)>
+    IndexUnitDataConsumerFactory;
+
+/// \brief Creates a frontend action that provides dependency, file inclusion
+/// and decl ocurrence information for the translation unit, and optionally its
+/// module dependencies.
+///
+/// Decl occurrence information is provided per-file, sorted by offset.
+///
+/// \param ConsumerFactory provides an \c IndexUnitDataConsumer to use for a
+/// translation unit.
+/// \param WrappedAction another frontend action to wrap over or null.
+std::unique_ptr<FrontendAction>
+createUnitIndexingAction(IndexUnitDataConsumerFactory ConsumerFactory,
+                         UnitIndexingOptions UnitIndexOpts,
+                         std::unique_ptr<FrontendAction> WrappedAction);
+
+/// Collects and provides dependency, file inclusion and decl occurrence
+/// information for a \c ModuleFile to an \c IndexUnitDataConsumer constructed
+/// from the provided \c IndexUnitDataConsumerFactory.
+void indexModuleFile(serialization::ModuleFile &Mod, const CompilerInstance &CI,
+                     IndexUnitDataConsumerFactory UnitConsumerFactory,
+                     UnitIndexingOptions Opts);
+
+} // namespace index
+} // namespace clang
+
+#endif
diff --git a/include/clang/module.modulemap b/include/clang/module.modulemap
index 4097ad2dc7..1625f284df 100644
--- a/include/clang/module.modulemap
+++ b/include/clang/module.modulemap
@@ -68,6 +68,7 @@ module Clang_Diagnostics {
   module Comment { header "AST/CommentDiagnostic.h" export * }
   module Driver { header "Driver/DriverDiagnostic.h" export * }
   module Frontend { header "Frontend/FrontendDiagnostic.h" export * }
+  module Index { header "Index/IndexDiagnostic.h" export * }
   module Lex { header "Lex/LexDiagnostic.h" export * }
   module Parse { header "Parse/ParseDiagnostic.h" export * }
   module Sema { header "Sema/SemaDiagnostic.h" export * }
diff --git a/lib/Basic/DiagnosticIDs.cpp b/lib/Basic/DiagnosticIDs.cpp
index ab7802651d..e67914ae80 100644
--- a/lib/Basic/DiagnosticIDs.cpp
+++ b/lib/Basic/DiagnosticIDs.cpp
@@ -89,6 +89,7 @@ VALIDATE_DIAG_SIZE(COMMENT)
 VALIDATE_DIAG_SIZE(SEMA)
 VALIDATE_DIAG_SIZE(ANALYSIS)
 VALIDATE_DIAG_SIZE(REFACTORING)
+VALIDATE_DIAG_SIZE(INDEX)
 #undef VALIDATE_DIAG_SIZE
 #undef STRINGIFY_NAME
 
@@ -114,6 +115,7 @@ static const StaticDiagInfoRec StaticDiagInfo[] = {
 #include "clang/Basic/DiagnosticSemaKinds.inc"
 #include "clang/Basic/DiagnosticAnalysisKinds.inc"
 #include "clang/Basic/DiagnosticRefactoringKinds.inc"
+#include "clang/Basic/DiagnosticIndexKinds.inc"
 #undef DIAG
 };
 
@@ -153,6 +155,7 @@ CATEGORY(CROSSTU, COMMENT)
 CATEGORY(SEMA, CROSSTU)
 CATEGORY(ANALYSIS, SEMA)
 CATEGORY(REFACTORING, ANALYSIS)
+CATEGORY(INDEX, REFACTORING)
 #undef CATEGORY
 
   // Avoid out of bounds reads.
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
index 325b233ac5..86d452711b 100644
--- a/lib/Driver/Driver.cpp
+++ b/lib/Driver/Driver.cpp
@@ -1234,7 +1234,9 @@ void Driver::generateCompilationDiagnostics(Compilation &C,
   }
 
   // Assume associated files are based off of the first temporary file.
-  CrashReportInfo CrashInfo(TempFiles[0], VFS);
+  CrashReportInfo CrashInfo(
+      TempFiles[0], VFS,
+      C.getArgs().getLastArgValue(options::OPT_index_store_path));
 
   std::string Script = CrashInfo.Filename.rsplit('.').first.str() + ".sh";
   std::error_code EC;
diff --git a/lib/Driver/Job.cpp b/lib/Driver/Job.cpp
index 765c05752d..5c556717d6 100644
--- a/lib/Driver/Job.cpp
+++ b/lib/Driver/Job.cpp
@@ -68,6 +68,8 @@ static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum,
     .Default(false);
   if (IsInclude)
     return HaveCrashVFS ? false : true;
+  if (StringRef(Flag).startswith("-index-store-path"))
+    return true;
 
   // The remaining flags are treated as a single argument.
 
@@ -206,6 +208,18 @@ rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx,
   IncFlags.push_back(std::move(NewInc));
 }
 
+/// Returns a path to a directory named \c DirName adjacent to the module
+/// cache directory:
+///  <...>.cache/vfs/<DirName>
+static llvm::SmallString<128>
+getDirAdjacentToModCache(StringRef DirName, CrashReportInfo *CrashInfo) {
+  llvm::SmallString<128> RelModCacheDir = llvm::sys::path::parent_path(
+      llvm::sys::path::parent_path(CrashInfo->VFSPath));
+  llvm::sys::path::append(RelModCacheDir, DirName);
+
+  return RelModCacheDir;
+}
+
 void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
                     CrashReportInfo *CrashInfo) const {
   // Always quote the exe.
@@ -220,6 +234,7 @@ void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
   }
 
   bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty();
+  bool HaveIndexStorePath = CrashInfo && !CrashInfo->IndexStorePath.empty();
   for (size_t i = 0, e = Args.size(); i < e; ++i) {
     const char *const Arg = Args[i];
 
@@ -267,14 +282,10 @@ void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
     OS << ' ';
     printArg(OS, CrashInfo->VFSPath.str(), Quote);
 
-    // The leftover modules from the crash are stored in
-    //  <name>.cache/vfs/modules
-    // Leave it untouched for pcm inspection and provide a clean/empty dir
-    // path to contain the future generated module cache:
-    //  <name>.cache/vfs/repro-modules
-    SmallString<128> RelModCacheDir = llvm::sys::path::parent_path(
-        llvm::sys::path::parent_path(CrashInfo->VFSPath));
-    llvm::sys::path::append(RelModCacheDir, "repro-modules");
+    // Provide an empty dir path for the future generated module cache to
+    // leave the leftover modules from the crash untouched for pcm inspection
+    SmallString<128> RelModCacheDir =
+        getDirAdjacentToModCache("repro-modules", CrashInfo);
 
     std::string ModCachePath = "-fmodules-cache-path=";
     ModCachePath.append(RelModCacheDir.c_str());
@@ -283,6 +294,22 @@ void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
     printArg(OS, ModCachePath, Quote);
   }
 
+  if (CrashInfo && HaveIndexStorePath) {
+    SmallString<128> IndexStoreDir;
+
+    if (HaveCrashVFS) {
+      // Provide a new index store, leaving the old one from the crash untouched
+      IndexStoreDir = getDirAdjacentToModCache("index-store", CrashInfo);
+    } else {
+      IndexStoreDir = "index-store";
+    }
+
+    OS << ' ';
+    printArg(OS, "-index-store-path", Quote);
+    OS << ' ';
+    printArg(OS, IndexStoreDir.c_str(), Quote);
+  }
+
   if (ResponseFile != nullptr) {
     OS << "\n Arguments passed via response file:\n";
     writeResponseFile(OS);
diff --git a/lib/Driver/ToolChains/Clang.cpp b/lib/Driver/ToolChains/Clang.cpp
index 484df52001..45ea603908 100644
--- a/lib/Driver/ToolChains/Clang.cpp
+++ b/lib/Driver/ToolChains/Clang.cpp
@@ -3611,6 +3611,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   RenderARCMigrateToolOptions(D, Args, CmdArgs);
 
+  if (Args.hasArg(options::OPT_index_store_path)) {
+    Args.AddLastArg(CmdArgs, options::OPT_index_store_path);
+    Args.AddLastArg(CmdArgs, options::OPT_index_ignore_system_symbols);
+    Args.AddLastArg(CmdArgs, options::OPT_index_record_codegen_name);
+
+    // If '-o' is passed along with '-fsyntax-only' pass it along the cc1
+    // invocation so that the index action knows what the out file is.
+    if (isa<CompileJobAction>(JA) && JA.getType() == types::TY_Nothing) {
+      Args.AddLastArg(CmdArgs, options::OPT_o);
+    }
+  }
+
   // Add preprocessing options like -I, -D, etc. if we are using the
   // preprocessor.
   //
diff --git a/lib/Driver/ToolChains/Darwin.cpp b/lib/Driver/ToolChains/Darwin.cpp
index 802b89b070..6434593390 100644
--- a/lib/Driver/ToolChains/Darwin.cpp
+++ b/lib/Driver/ToolChains/Darwin.cpp
@@ -436,6 +436,10 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // more information.
   ArgStringList CmdArgs;
 
+  Args.ClaimAllArgs(options::OPT_index_store_path);
+  Args.ClaimAllArgs(options::OPT_index_ignore_system_symbols);
+  Args.ClaimAllArgs(options::OPT_index_record_codegen_name);
+
   /// Hack(tm) to ignore linking errors when we are doing ARC migration.
   if (Args.hasArg(options::OPT_ccc_arcmt_check,
                   options::OPT_ccc_arcmt_migrate)) {
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index 7208177aa0..5f5c8377fd 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -28,6 +28,7 @@
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Frontend/VerifyDiagnosticConsumer.h"
+#include "clang/Index/IndexingAction.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/PTHManager.h"
 #include "clang/Lex/Preprocessor.h"
@@ -1148,6 +1149,10 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
   SourceMgr.pushModuleBuildStack(ModuleName,
     FullSourceLoc(ImportLoc, ImportingInstance.getSourceManager()));
 
+  // Pass along the GenModuleActionWrapper callback
+  auto WrapGenModuleAction = ImportingInstance.getGenModuleActionWrapper();
+  Instance.setGenModuleActionWrapper(WrapGenModuleAction);
+
   // If we're collecting module dependencies, we need to share a collector
   // between all of the module CompilerInstances. Other than that, we don't
   // want to produce any dependency output from the module build.
@@ -1166,8 +1171,11 @@ compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
   llvm::CrashRecoveryContext CRC;
   CRC.RunSafelyOnThread(
       [&]() {
-        GenerateModuleFromModuleMapAction Action;
-        Instance.ExecuteAction(Action);
+        std::unique_ptr<FrontendAction> Action(
+            new GenerateModuleFromModuleMapAction);
+        if (WrapGenModuleAction)
+          Action = WrapGenModuleAction(FrontendOpts, std::move(Action));
+        Instance.ExecuteAction(*Action);
       },
       ThreadStackSize);
 
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index 833039e693..53bae9a83f 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -1488,6 +1488,10 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       << "ARC migration" << "ObjC migration";
   }
 
+  Opts.IndexStorePath = Args.getLastArgValue(OPT_index_store_path);
+  Opts.IndexIgnoreSystemSymbols = Args.hasArg(OPT_index_ignore_system_symbols);
+  Opts.IndexRecordCodegenName = Args.hasArg(OPT_index_record_codegen_name);
+
   InputKind DashX(InputKind::Unknown);
   if (const Arg *A = Args.getLastArg(OPT_x)) {
     StringRef XValue = A->getValue();
diff --git a/lib/FrontendTool/CMakeLists.txt b/lib/FrontendTool/CMakeLists.txt
index 7e11be0ce4..4a713e5902 100644
--- a/lib/FrontendTool/CMakeLists.txt
+++ b/lib/FrontendTool/CMakeLists.txt
@@ -8,6 +8,7 @@ set(link_libs
   clangCodeGen
   clangDriver
   clangFrontend
+  clangIndex
   clangRewriteFrontend
   )
 
diff --git a/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 4167e1fe20..2c3b033c5a 100644
--- a/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -23,6 +23,7 @@
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/FrontendPluginRegistry.h"
 #include "clang/Frontend/Utils.h"
+#include "clang/Index/RecordingAction.h"
 #include "clang/Rewrite/Frontend/FrontendActions.h"
 #include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
 #include "llvm/Option/OptTable.h"
@@ -164,6 +165,22 @@ CreateFrontendAction(CompilerInstance &CI) {
   }
 #endif
 
+  if (!FEOpts.IndexStorePath.empty()) {
+    auto WrapWithIndexRecordAction =
+        [&](const FrontendOptions &opts,
+            std::unique_ptr<FrontendAction> WrappedAction) {
+          auto RecordOpts =
+              index::getRecordingOptionsFromFrontendOptions(FEOpts);
+          return index::createIndexDataRecordingAction(
+              RecordOpts, std::move(WrappedAction));
+        };
+
+    // Wrap the main action as well as any GenerateModuleActions created while
+    // loading modules
+    Act = WrapWithIndexRecordAction(FEOpts, std::move(Act));
+    CI.setGenModuleActionWrapper(WrapWithIndexRecordAction);
+  }
+
   // If there are any AST files to merge, create a frontend action
   // adaptor to perform the merge.
   if (!FEOpts.ASTMergeFiles.empty())
diff --git a/lib/Index/CMakeLists.txt b/lib/Index/CMakeLists.txt
index c9fbfafcf9..38d84e4c19 100644
--- a/lib/Index/CMakeLists.txt
+++ b/lib/Index/CMakeLists.txt
@@ -6,12 +6,14 @@ set(LLVM_LINK_COMPONENTS
 add_clang_library(clangIndex
   CodegenNameGenerator.cpp
   CommentToXML.cpp
+  FileIndexData.cpp
   IndexBody.cpp
   IndexDecl.cpp
   IndexingAction.cpp
   IndexingContext.cpp
   IndexSymbol.cpp
   IndexTypeSourceInfo.cpp
+  UnitIndexDataRecorder.cpp
   USRGeneration.cpp
 
   ADDITIONAL_HEADERS
@@ -23,6 +25,7 @@ add_clang_library(clangIndex
   clangBasic
   clangFormat
   clangFrontend
+  clangLex
   clangRewrite
   clangSerialization
   clangToolingCore
diff --git a/lib/Index/FileIndexData.cpp b/lib/Index/FileIndexData.cpp
new file mode 100644
index 0000000000..b68319a777
--- /dev/null
+++ b/lib/Index/FileIndexData.cpp
@@ -0,0 +1,52 @@
+//===--- FileIndexData.cpp - Index data per file ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "FileIndexData.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclTemplate.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Path.h"
+
+using namespace clang;
+using namespace clang::index;
+
+void FileIndexData::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset,
+                                     const Decl *D,
+                                     ArrayRef<SymbolRelation> Relations) {
+  assert(D->isCanonicalDecl() &&
+         "Occurrences should be associated with their canonical decl");
+
+  Decls.emplace_back(Roles, Offset, D, Relations);
+}
+
+std::vector<DeclOccurrence>
+FileIndexData::getDeclOccurrencesSortedByOffset() const {
+  std::vector<DeclOccurrence> Sorted(Decls);
+  std::sort(Sorted.begin(), Sorted.end());
+  return Sorted;
+}
+
+void FileIndexData::print(llvm::raw_ostream &OS) const {
+  OS << "DECLS BEGIN ---\n";
+  for (auto &DclInfo : Decls) {
+    auto D = DclInfo.Dcl;
+    SourceManager &SM = D->getASTContext().getSourceManager();
+    SourceLocation Loc = SM.getFileLoc(D->getLocation());
+    PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+    OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine()
+       << ':' << PLoc.getColumn();
+
+    if (auto ND = dyn_cast<NamedDecl>(D)) {
+      OS << ' ' << ND->getNameAsString();
+    }
+
+    OS << '\n';
+  }
+  OS << "DECLS END ---\n";
+}
diff --git a/lib/Index/FileIndexData.h b/lib/Index/FileIndexData.h
new file mode 100644
index 0000000000..386b785b51
--- /dev/null
+++ b/lib/Index/FileIndexData.h
@@ -0,0 +1,56 @@
+//===--- FileIndexData.h - Index data per file --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H
+#define LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H
+
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Index/DeclOccurrence.h"
+#include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include <vector>
+
+namespace clang {
+class IdentifierInfo;
+
+namespace index {
+
+/// Stores the declaration occurrences seen in a particular source or header
+/// file of a translation unit
+class FileIndexData {
+private:
+  FileID FID;
+  bool IsSystem;
+  std::vector<DeclOccurrence> Decls;
+
+public:
+  FileIndexData(FileID FID, bool IsSystem) : FID(FID), IsSystem(IsSystem) {}
+
+  std::vector<DeclOccurrence> getDeclOccurrencesSortedByOffset() const;
+
+  FileID getFileID() const { return FID; }
+  bool isSystem() const { return IsSystem; }
+
+  /// Adds an occurrence of the canonical declaration \c D at the supplied
+  /// \c Offset
+  ///
+  /// \param Roles the roles the occurrence fulfills in this position.
+  /// \param Offset the offset in the file of this occurrence.
+  /// \param D the canonical declaration this is an occurrence of.
+  /// \param Relations the set of symbols related to this occurrence.
+  void addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, const Decl *D,
+                        ArrayRef<SymbolRelation> Relations);
+  void print(llvm::raw_ostream &OS) const;
+};
+
+} // end namespace index
+} // end namespace clang
+
+#endif
diff --git a/lib/Index/IndexingAction.cpp b/lib/Index/IndexingAction.cpp
index 411657bf3d..e17b369e9d 100644
--- a/lib/Index/IndexingAction.cpp
+++ b/lib/Index/IndexingAction.cpp
@@ -8,35 +8,50 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Index/IndexingAction.h"
+#include "clang/Index/RecordingAction.h"
+#include "clang/Index/UnitIndexingAction.h"
+
+#include "FileIndexData.h"
 #include "IndexingContext.h"
+#include "UnitIndexDataRecorder.h"
+#include "clang/Basic/FileManager.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendAction.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/MultiplexConsumer.h"
+#include "clang/Frontend/Utils.h"
 #include "clang/Index/IndexDataConsumer.h"
+#include "clang/Index/IndexDiagnostic.h"
+#include "clang/Index/UnitIndexDataConsumer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Serialization/ASTReader.h"
+#include "llvm/Support/Path.h"
 
 using namespace clang;
 using namespace clang::index;
 
+void UnitIndexDataConsumer::_anchor() {}
 void IndexDataConsumer::_anchor() {}
 
 bool IndexDataConsumer::handleDeclOccurence(const Decl *D, SymbolRoleSet Roles,
                                             ArrayRef<SymbolRelation> Relations,
                                             FileID FID, unsigned Offset,
+                                            bool IsInSystemFile,
                                             ASTNodeInfo ASTNode) {
   return true;
 }
 
 bool IndexDataConsumer::handleMacroOccurence(const IdentifierInfo *Name,
                                              const MacroInfo *MI, SymbolRoleSet Roles,
-                                             FileID FID, unsigned Offset) {
+                                             FileID FID, unsigned Offset,
+                                             bool IsInSystemFile) {
   return true;
 }
 
 bool IndexDataConsumer::handleModuleOccurence(const ImportDecl *ImportD,
                                               SymbolRoleSet Roles,
-                                              FileID FID, unsigned Offset) {
+                                              FileID FID, unsigned Offset,
+                                              bool IsInSystemFile) {
   return true;
 }
 
@@ -73,96 +88,116 @@ protected:
   }
 };
 
-class IndexActionBase {
-protected:
-  std::shared_ptr<IndexDataConsumer> DataConsumer;
-  IndexingContext IndexCtx;
-
-  IndexActionBase(std::shared_ptr<IndexDataConsumer> dataConsumer,
-                  IndexingOptions Opts)
-    : DataConsumer(std::move(dataConsumer)),
-      IndexCtx(Opts, *DataConsumer) {}
-
-  std::unique_ptr<IndexASTConsumer>
-  createIndexASTConsumer(CompilerInstance &CI) {
-    return llvm::make_unique<IndexASTConsumer>(CI.getPreprocessorPtr(),
-                                               IndexCtx);
-  }
-
-  void finish() {
-    DataConsumer->finish();
-  }
+/// Abstracts the core logic shared between \c IndexAction and
+/// \c WrappingIndexAction frontend actions.
+class IndexActionImpl {
+public:
+  virtual ~IndexActionImpl() = default;
+
+  /// Called at the beginning of processing a single input, this creates the
+  /// IndexASTConsumer object to use.
+  ///
+  /// \param CI The compiler instance used to process the input
+  /// \returns the created IndexASTConsumer.
+  virtual std::unique_ptr<IndexASTConsumer>
+  createIndexASTConsumer(CompilerInstance &CI) = 0;
+
+  /// Callback at the end of processing a single input.
+  ///
+  /// \param CI The compiler instance used to process the input. It will be the
+  /// same instance as provided in \c createIndexASTConsumer.
+  virtual void finish(CompilerInstance &CI) = 0;
 };
 
-class IndexAction : public ASTFrontendAction, IndexActionBase {
+class IndexAction : public ASTFrontendAction {
+  std::unique_ptr<IndexActionImpl> Impl;
+
 public:
-  IndexAction(std::shared_ptr<IndexDataConsumer> DataConsumer,
-              IndexingOptions Opts)
-    : IndexActionBase(std::move(DataConsumer), Opts) {}
+  IndexAction(std::unique_ptr<IndexActionImpl> Impl) : Impl(std::move(Impl)) {}
 
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override {
-    return createIndexASTConsumer(CI);
+    return Impl->createIndexASTConsumer(CI);
   }
 
   void EndSourceFileAction() override {
     FrontendAction::EndSourceFileAction();
-    finish();
+    Impl->finish(getCompilerInstance());
   }
 };
 
-class WrappingIndexAction : public WrapperFrontendAction, IndexActionBase {
-  bool IndexActionFailed = false;
+class WrappingIndexAction : public WrapperFrontendAction {
+  std::unique_ptr<IndexActionImpl> Impl;
+  bool CreatedASTConsumer = false;
 
 public:
   WrappingIndexAction(std::unique_ptr<FrontendAction> WrappedAction,
-                      std::shared_ptr<IndexDataConsumer> DataConsumer,
-                      IndexingOptions Opts)
-    : WrapperFrontendAction(std::move(WrappedAction)),
-      IndexActionBase(std::move(DataConsumer), Opts) {}
+                      std::unique_ptr<IndexActionImpl> Impl)
+      : WrapperFrontendAction(std::move(WrappedAction)), Impl(std::move(Impl)) {
+  }
 
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
-                                                 StringRef InFile) override;
-  void EndSourceFileAction() override;
+                                                 StringRef InFile) override {
+    auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
+    if (!OtherConsumer)
+      return nullptr;
+
+    std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+    Consumers.push_back(std::move(OtherConsumer));
+    Consumers.push_back(Impl->createIndexASTConsumer(CI));
+    CreatedASTConsumer = true;
+
+    return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
+  };
+
+  void EndSourceFileAction() override {
+    // Invoke wrapped action's method.
+    WrapperFrontendAction::EndSourceFileAction();
+    if (CreatedASTConsumer) {
+      CreatedASTConsumer = false;
+      Impl->finish(getCompilerInstance());
+    }
+  };
 };
 
-} // anonymous namespace
+/// An implementation for \c IndexAction or \c WrappingIndexAction that provides
+/// decl ocurrences information from the AST.
+class DataConsumerActionImpl : public IndexActionImpl {
+protected:
+  std::shared_ptr<IndexDataConsumer> DataConsumer;
+  IndexingContext IndexCtx;
 
-void WrappingIndexAction::EndSourceFileAction() {
-  // Invoke wrapped action's method.
-  WrapperFrontendAction::EndSourceFileAction();
-  if (!IndexActionFailed)
-    finish();
-}
+public:
+  DataConsumerActionImpl(std::shared_ptr<IndexDataConsumer> Consumer,
+                         IndexingOptions Opts)
+      : DataConsumer(std::move(Consumer)), IndexCtx(Opts, *DataConsumer) {}
 
-std::unique_ptr<ASTConsumer>
-WrappingIndexAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
-  auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
-  if (!OtherConsumer) {
-    IndexActionFailed = true;
-    return nullptr;
+  std::unique_ptr<IndexASTConsumer>
+  createIndexASTConsumer(CompilerInstance &CI) override {
+    IndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot);
+    return llvm::make_unique<IndexASTConsumer>(CI.getPreprocessorPtr(),
+                                               IndexCtx);
   }
 
-  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
-  Consumers.push_back(std::move(OtherConsumer));
-  Consumers.push_back(createIndexASTConsumer(CI));
-  return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
-}
+  void finish(CompilerInstance &CI) override { DataConsumer->finish(); }
+};
+
+} // anonymous namespace
 
 std::unique_ptr<FrontendAction>
 index::createIndexingAction(std::shared_ptr<IndexDataConsumer> DataConsumer,
                             IndexingOptions Opts,
                             std::unique_ptr<FrontendAction> WrappedAction) {
+  auto ActionImpl =
+      llvm::make_unique<DataConsumerActionImpl>(std::move(DataConsumer), Opts);
   if (WrappedAction)
     return llvm::make_unique<WrappingIndexAction>(std::move(WrappedAction),
-                                                  std::move(DataConsumer),
-                                                  Opts);
-  return llvm::make_unique<IndexAction>(std::move(DataConsumer), Opts);
+                                                  std::move(ActionImpl));
+  return llvm::make_unique<IndexAction>(std::move(ActionImpl));
 }
 
-
 static bool topLevelDeclVisitor(void *context, const Decl *D) {
   IndexingContext &IndexCtx = *static_cast<IndexingContext*>(context);
   return IndexCtx.indexTopLevelDecl(D);
@@ -209,3 +244,551 @@ void index::indexModuleFile(serialization::ModuleFile &Mod,
   }
   DataConsumer->finish();
 }
+
+//===----------------------------------------------------------------------===//
+// Index Data Recording
+//===----------------------------------------------------------------------===//
+
+/// Construct a \c UnitDetails for a translation unit with the provided root
+/// \c FileEntry or \c Module and with the provided sysroot path.
+static index::UnitDetails getUnitDetails(const CompilerInstance &CI,
+                                         std::string OutputFile,
+                                         const FileEntry *RootFile,
+                                         Module *UnitMod,
+                                         StringRef SysrootPath) {
+  std::string ModuleName =
+      UnitMod ? UnitMod->getFullModuleName() : std::string();
+  bool IsSystemUnit = UnitMod ? UnitMod->IsSystem : false;
+  bool IsModuleUnit = UnitMod != nullptr;
+  bool IsDebugCompilation = CI.getCodeGenOpts().OptimizationLevel == 0;
+
+  // Ignore sysroot path if it points to root, otherwise every header will be
+  // treated as system one.
+  if (llvm::sys::path::root_path(SysrootPath) == SysrootPath)
+    SysrootPath = "";
+
+  return {CI,           UnitMod,      ModuleName,
+          RootFile,     OutputFile,   SysrootPath,
+          IsSystemUnit, IsModuleUnit, IsDebugCompilation};
+}
+
+/// Construct a \c UnitDetails from the invocation associated with the provided
+/// \c CompilerInstance and the provided sysroot path.
+static index::UnitDetails getUnitDetails(const CompilerInstance &CI,
+                                         StringRef SysrootPath) {
+  SourceManager &SM = CI.getASTContext().getSourceManager();
+
+  std::string OutputFile = CI.getFrontendOpts().OutputFile;
+  if (OutputFile.empty()) {
+    OutputFile = CI.getFrontendOpts().Inputs[0].getFile();
+    OutputFile += ".o";
+  }
+
+  const FileEntry *RootFile = nullptr;
+  Module *UnitMod = nullptr;
+  bool IsModuleGeneration = CI.getLangOpts().isCompilingModule();
+  if (!IsModuleGeneration &&
+      CI.getFrontendOpts().ProgramAction != frontend::GeneratePCH)
+    RootFile = SM.getFileEntryForID(SM.getMainFileID());
+
+  if (IsModuleGeneration) {
+    HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
+    UnitMod = HS.lookupModule(CI.getLangOpts().CurrentModule,
+                              /*AllowSearch=*/false);
+    assert(UnitMod && "only loaded modules should be indexed");
+  }
+  return getUnitDetails(CI, std::move(OutputFile), RootFile, UnitMod,
+                        SysrootPath);
+}
+
+/// Construct a \c UnitDetails for the given module file.
+static index::UnitDetails getUnitDetails(serialization::ModuleFile &Mod,
+                                         const CompilerInstance &CI,
+                                         StringRef SysrootPath) {
+  HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
+  Module *UnitMod = HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false);
+  assert(UnitMod && "only loaded modules should be indexed");
+
+  return getUnitDetails(CI, /*OutputFile=*/Mod.FileName, /*RootFile=*/nullptr,
+                        UnitMod, SysrootPath);
+}
+
+namespace {
+
+/// Collects and groups consumed index data by \c FileID.
+class FileIndexDataCollector : public IndexDataConsumer {
+  std::shared_ptr<Preprocessor> PP;
+  typedef llvm::DenseMap<FileID, std::unique_ptr<FileIndexData>>
+      IndexDataByFileTy;
+  IndexDataByFileTy IndexDataByFile;
+
+public:
+  void setPreprocessor(std::shared_ptr<Preprocessor> PreProc) override {
+    PP = PreProc;
+  }
+
+  IndexDataByFileTy::const_iterator begin() const {
+    return IndexDataByFile.begin();
+  }
+
+  IndexDataByFileTy::const_iterator end() const {
+    return IndexDataByFile.end();
+  }
+
+  bool empty() const { return IndexDataByFile.empty(); }
+
+  bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles,
+                           ArrayRef<SymbolRelation> Relations,
+                           FileID FID, unsigned Offset, bool IsInSystemFile,
+                           ASTNodeInfo ASTNode) override {
+    ASTContext &Ctx = D->getASTContext();
+    SourceManager &SM = Ctx.getSourceManager();
+
+    // Ignore occurrences in the predefines buffer
+    if (FID == PP->getPredefinesFileID())
+      return true;
+
+    FileIndexData &FileData = getFileIndexData(FID, IsInSystemFile);
+    FileData.addDeclOccurence(Roles, Offset, D, Relations);
+    return true;
+  }
+
+private:
+  FileIndexData &getFileIndexData(FileID FID, bool IsInSystemFile) {
+    auto &Entry = IndexDataByFile[FID];
+    if (!Entry) {
+      Entry.reset(new FileIndexData(FID, IsInSystemFile));
+    }
+    return *Entry;
+  }
+};
+
+struct IncludeLocation {
+  const FileEntry *Source;
+  const FileEntry *Target;
+  unsigned Line;
+};
+
+/// Preprocessor callbacks to collect file to file inclusion information
+class IncludePPCallbacks : public PPCallbacks {
+  SystemFileCache &SystemCache;
+  UnitIndexingOptions::FileIncludeFilterKind FileIncludeFilter;
+  std::vector<IncludeLocation> &Includes;
+  SourceManager &SourceMgr;
+
+public:
+  IncludePPCallbacks(SystemFileCache &SystemCache,
+                     UnitIndexingOptions::FileIncludeFilterKind IncludeFilter,
+                     std::vector<IncludeLocation> &IncludesForFile,
+                     SourceManager &SourceMgr)
+      : SystemCache(SystemCache), FileIncludeFilter(IncludeFilter),
+        Includes(IncludesForFile), SourceMgr(SourceMgr) {}
+
+  virtual void InclusionDirective(SourceLocation HashLoc,
+                                  const Token &IncludeTok,
+                                  StringRef FileName,
+                                  bool IsAngled,
+                                  CharSourceRange FilenameRange,
+                                  const FileEntry *File,
+                                  StringRef SearchPath,
+                                  StringRef RelativePath,
+                                  const Module *Imported) override {
+    if (HashLoc.isFileID() && File && File->isValid())
+      addInclude(HashLoc, File);
+  }
+
+private:
+  void addInclude(SourceLocation From, const FileEntry *To) {
+    assert(To);
+    if (FileIncludeFilter == UnitIndexingOptions::FileIncludeFilterKind::None)
+      return;
+
+    std::pair<FileID, unsigned> LocInfo =
+        SourceMgr.getDecomposedExpansionLoc(From);
+
+    if (LocInfo.first.isInvalid())
+      return; // Ignore invalid locations.
+
+    if (FileIncludeFilter ==
+        UnitIndexingOptions::FileIncludeFilterKind::UserOnly)
+      if (SystemCache.isSystem(LocInfo.first, SourceMgr))
+        return; // Ignore includes of system headers.
+
+    if (auto *FE = SourceMgr.getFileEntryForID(LocInfo.first)) {
+      auto lineNo = SourceMgr.getLineNumber(LocInfo.first, LocInfo.second);
+      Includes.push_back({FE, To, lineNo});
+    }
+  }
+};
+
+/// Abstract interface for providing the file and module dependencies of a
+/// translation unit, as well as the set of file to file inclusions
+class IndexDependencyProvider {
+public:
+  virtual ~IndexDependencyProvider() {}
+
+  virtual void forEachFileDependency(
+      const CompilerInstance &CI,
+      llvm::function_ref<void(const FileEntry *FE, bool IsSystem)> Callback)
+      const = 0;
+
+  virtual void
+  forEachInclude(llvm::function_ref<void(const FileEntry *Source, unsigned Line,
+                                         const FileEntry *Target)>
+                     Callback) const = 0;
+  virtual void forEachModuleImport(
+      const CompilerInstance &CI,
+      llvm::function_ref<void(serialization::ModuleFile &Mod, bool IsSystem)>
+          Callback) const = 0;
+};
+
+/// An IndexDependencyProvider for the index data collected by
+/// \c FileIndexDependencyCollector.
+class FileIndexDependencyProvider : public IndexDependencyProvider {
+  llvm::SetVector<const FileEntry *> Files;
+  llvm::BitVector IsSystemByUID;
+  std::vector<IncludeLocation> Includes;
+  bool IncludeSysModules;
+
+public:
+  FileIndexDependencyProvider(llvm::SetVector<const FileEntry *> Entries,
+                              llvm::BitVector IsSystemByUID,
+                              std::vector<IncludeLocation> Includes,
+                              bool IncludeSysDeps)
+      : Files(std::move(Entries)), IsSystemByUID(std::move(IsSystemByUID)),
+        Includes(std::move(Includes)), IncludeSysModules(IncludeSysDeps) {}
+
+  void forEachFileDependency(
+      const CompilerInstance &CI,
+      llvm::function_ref<void(const FileEntry *FE, bool IsSystem)> Callback)
+      const override {
+    for (auto *FE : Files)
+      Callback(FE, isSystemFile(FE));
+  }
+
+  void
+  forEachInclude(llvm::function_ref<void(const FileEntry *Source, unsigned Line,
+                                         const FileEntry *Target)>
+                     Callback) const override {
+    for (auto &Include : Includes)
+      Callback(Include.Source, Include.Line, Include.Target);
+  }
+
+  void forEachModuleImport(
+      const CompilerInstance &CI,
+      llvm::function_ref<void(serialization::ModuleFile &Mod, bool IsSystem)>
+          Callback) const override {
+    HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
+
+    if (auto Reader = CI.getModuleManager()) {
+      Reader->getModuleManager().visit(
+          [&](serialization::ModuleFile &Mod) -> bool {
+            bool IsSystemMod = false;
+            if (Mod.isModule()) {
+              if (auto *M =
+                      HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false))
+                IsSystemMod = M->IsSystem;
+            }
+            if (!IsSystemMod || IncludeSysModules)
+              Callback(Mod, IsSystemMod);
+            return true; // skip module dependencies.
+          });
+    }
+  }
+
+private:
+  bool isSystemFile(const FileEntry *FE) const {
+    auto UID = FE->getUID();
+    return IsSystemByUID.size() > UID && IsSystemByUID[UID];
+  }
+};
+
+/// Collects file and module dependency information for a translation unit,
+/// including file to file inclusions.
+class FileIndexDependencyCollector : public DependencyCollector {
+  SystemFileCache &SystemCache;
+  UnitIndexingOptions IndexOpts;
+  llvm::SetVector<const FileEntry *> SeenFiles;
+  llvm::BitVector IsSystemByUID;
+  std::vector<IncludeLocation> Includes;
+  SourceManager *SourceMgr = nullptr;
+
+public:
+  FileIndexDependencyCollector(SystemFileCache &SystemCache,
+                               UnitIndexingOptions IndexOpts)
+      : SystemCache(SystemCache), IndexOpts(IndexOpts) {}
+
+  void attachToPreprocessor(Preprocessor &PP) override {
+    DependencyCollector::attachToPreprocessor(PP);
+    PP.addPPCallbacks(llvm::make_unique<IncludePPCallbacks>(
+        SystemCache, IndexOpts.FileIncludeFilter, Includes,
+        PP.getSourceManager()));
+  }
+
+  void setSourceManager(SourceManager *SourceMgr) {
+    this->SourceMgr = SourceMgr;
+  }
+
+  FileIndexDependencyProvider consume() {
+    return FileIndexDependencyProvider(
+        std::move(SeenFiles), std::move(IsSystemByUID), std::move(Includes),
+        IndexOpts.IncludeSystemDependencies);
+  }
+
+private:
+  bool needSystemDependencies() override {
+    return IndexOpts.IncludeSystemDependencies;
+  }
+
+  bool sawDependency(StringRef Filename, bool FromModule, bool IsSystem,
+                     bool IsModuleFile, bool IsMissing) override {
+    bool SawIt = DependencyCollector::sawDependency(
+        Filename, FromModule, IsSystem, IsModuleFile, IsMissing);
+    if (auto *FE = SourceMgr->getFileManager().getFile(Filename)) {
+      if (SawIt)
+        SeenFiles.insert(FE);
+
+      // Record system-ness for all files that we pass through.
+      if (IsSystemByUID.size() < FE->getUID() + 1)
+        IsSystemByUID.resize(FE->getUID() + 1);
+      IsSystemByUID[FE->getUID()] = IsSystem || isInSysroot(Filename);
+    }
+    return SawIt;
+  }
+
+  bool isInSysroot(StringRef Filename) {
+    StringRef SysrootPath = SystemCache.getSysrootPath();
+    return !SysrootPath.empty() && Filename.startswith(SysrootPath);
+  }
+};
+} // anonymous namespace
+
+static void reportData(const CompilerInstance &CI,
+                       const FileIndexDataCollector &Collector,
+                       const IndexDependencyProvider &DepProvider,
+                       UnitDetails UnitInfo,
+                       const IndexUnitDataConsumerFactory &UnitConsumerFactory,
+                       const UnitIndexingOptions &IndexOpts) {
+
+  std::unique_ptr<UnitIndexDataConsumer> Consumer =
+      UnitConsumerFactory(UnitInfo);
+  if (!Consumer)
+    return;
+
+  DepProvider.forEachFileDependency(
+      CI, [&](const FileEntry *FE, bool IsSystemFile) {
+        Consumer->handleFileDependency(FE, IsSystemFile);
+      });
+  DepProvider.forEachInclude(
+      [&](const FileEntry *Source, unsigned Line, const FileEntry *Target) {
+        Consumer->handleInclude(Source, Line, Target);
+      });
+  DepProvider.forEachModuleImport(
+      CI, [&](serialization::ModuleFile &Mod, bool IsSystemMod) {
+        Consumer->handleModuleImport(Mod, IsSystemMod);
+        if (Mod.isModule() && Consumer->shouldIndexModuleDependency(Mod))
+          indexModuleFile(Mod, CI, UnitConsumerFactory, IndexOpts);
+      });
+
+  for (auto I = Collector.begin(), E = Collector.end(); I != E; ++I) {
+    FileID FID = I->first;
+    const FileIndexData &FileData = *I->second;
+    if (Consumer->handleFileOccurrences(
+            FID, FileData.getDeclOccurrencesSortedByOffset(),
+            FileData.isSystem()))
+      return;
+  }
+
+  Consumer->finish();
+}
+
+namespace {
+
+/// An implementation for IndexAction or WrappingIndexAction that gathers decl
+/// occurrence, file inclusion and dependency information for the translation
+/// unit and, optionally, its module dependencies.
+class UnitDataConsumerActionImpl : public IndexActionImpl {
+  UnitIndexingOptions IndexOpts;
+  FileIndexDataCollector Collector;
+  IndexingContext IndexCtx;
+  FileIndexDependencyCollector DepCollector;
+  IndexUnitDataConsumerFactory UnitConsumerFactory;
+
+public:
+  UnitDataConsumerActionImpl(UnitIndexingOptions UnitIndexOpts,
+                             IndexUnitDataConsumerFactory UnitConsumerFactory)
+      : IndexOpts(UnitIndexOpts), IndexCtx(UnitIndexOpts, Collector),
+        DepCollector(IndexCtx.getSystemCache(), IndexOpts),
+        UnitConsumerFactory(std::move(UnitConsumerFactory)) {}
+
+  std::unique_ptr<IndexASTConsumer>
+  createIndexASTConsumer(CompilerInstance &CI) override {
+    IndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot);
+
+    std::shared_ptr<Preprocessor> PP = CI.getPreprocessorPtr();
+    Collector.setPreprocessor(PP);
+    DepCollector.setSourceManager(&CI.getSourceManager());
+    DepCollector.attachToPreprocessor(CI.getPreprocessor());
+
+    return llvm::make_unique<IndexASTConsumer>(PP, IndexCtx);
+  }
+
+  /// Provides the collected indexing info to the \c IndexUnitDataConsumer
+  void finish(CompilerInstance &CI) override {
+    // The consumer may emit more diagnostics so do the begin/end source file
+    // invocations on the diagnostic client.
+    // FIXME: FrontendAction::EndSourceFile() should probably not call
+    // CI.getDiagnosticClient().EndSourceFile()' until after it has called
+    // 'EndSourceFileAction()', so that code executing during
+    // EndSourceFileAction() can emit diagnostics. If this is fixed,
+    // DiagClientBeginEndRAII can go away.
+    struct DiagClientBeginEndRAII {
+      CompilerInstance &CI;
+      DiagClientBeginEndRAII(CompilerInstance &CI) : CI(CI) {
+        CI.getDiagnosticClient().BeginSourceFile(CI.getLangOpts());
+      }
+      ~DiagClientBeginEndRAII() { CI.getDiagnosticClient().EndSourceFile(); }
+    } diagClientBeginEndRAII(CI);
+
+    Collector.finish();
+    reportData(CI, Collector, DepCollector.consume(),
+               getUnitDetails(CI, IndexCtx.getSysrootPath()),
+               UnitConsumerFactory, IndexOpts);
+  }
+};
+
+/// Provides the file and module dependency information for a \c ModuleFile
+class ModuleFileIndexDependencyCollector : public IndexDependencyProvider {
+  serialization::ModuleFile &ModFile;
+  bool CollectSystemDependencies;
+
+public:
+  ModuleFileIndexDependencyCollector(serialization::ModuleFile &Mod,
+                                     bool CollectSystemDependencies)
+      : ModFile(Mod), CollectSystemDependencies(CollectSystemDependencies) {}
+
+  void forEachFileDependency(
+      const CompilerInstance &CI,
+      llvm::function_ref<void(const FileEntry *FE, bool IsSystem)> Callback)
+      const override {
+    auto Reader = CI.getModuleManager();
+    Reader->visitInputFiles(
+        ModFile, CollectSystemDependencies, /*Complain=*/false,
+        [&](const serialization::InputFile &IF, bool IsSystem) {
+          auto *FE = IF.getFile();
+          if (!FE)
+            return;
+          // Ignore module map files, they are not as important to track as
+          // source files and they may be auto-generated which would create an
+          // undesirable dependency on an intermediate build byproduct.
+          if (FE->getName().endswith("module.modulemap"))
+            return;
+
+          Callback(FE, IsSystem);
+        });
+  }
+
+  void
+  forEachInclude(llvm::function_ref<void(const FileEntry *Source, unsigned Line,
+                                         const FileEntry *Target)>
+                     Callback) const override {
+    // FIXME: Module files without a preprocessing record do not have info about
+    // include locations. Serialize enough data to be able to retrieve such
+    // info.
+  }
+
+  void forEachModuleImport(
+      const CompilerInstance &CI,
+      llvm::function_ref<void(serialization::ModuleFile &Mod, bool IsSystem)>
+          Callback) const override {
+    HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo();
+    for (auto *Mod : ModFile.Imports) {
+      bool IsSystemMod = false;
+      if (auto *M = HS.lookupModule(Mod->ModuleName, /*AllowSearch=*/false))
+        IsSystemMod = M->IsSystem;
+      if (!IsSystemMod || CollectSystemDependencies)
+        Callback(*Mod, IsSystemMod);
+    }
+  }
+};
+} // anonymous namespace.
+
+void index::indexModuleFile(serialization::ModuleFile &Mod,
+                            const CompilerInstance &CI,
+                            IndexUnitDataConsumerFactory UnitConsumerFactory,
+                            UnitIndexingOptions IndexOpts) {
+
+  DiagnosticsEngine &Diag = CI.getDiagnostics();
+  Diag.Report(Mod.ImportLoc, diag::remark_index_producing_module_file_data)
+      << Mod.FileName;
+
+  FileIndexDataCollector Collector;
+  IndexingContext ModIndexCtx(IndexOpts, Collector);
+
+  auto &ASTCtx = CI.getASTContext();
+  Collector.initialize(ASTCtx);
+  Collector.setPreprocessor(CI.getPreprocessorPtr());
+  ModIndexCtx.setASTContext(ASTCtx);
+  ModIndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot);
+
+  for (const Decl *D : CI.getModuleManager()->getModuleFileLevelDecls(Mod))
+    ModIndexCtx.indexTopLevelDecl(D);
+
+  Collector.finish();
+
+  ModuleFileIndexDependencyCollector DepCollector(
+      Mod, IndexOpts.IncludeSystemDependencies);
+
+  reportData(CI, Collector, DepCollector,
+             getUnitDetails(Mod, CI, ModIndexCtx.getSysrootPath()),
+             UnitConsumerFactory, IndexOpts);
+}
+
+std::unique_ptr<FrontendAction>
+index::createUnitIndexingAction(IndexUnitDataConsumerFactory ConsumerFactory,
+                                UnitIndexingOptions IndexOpts,
+                                std::unique_ptr<FrontendAction> WrappedAction) {
+  auto ActionImpl = llvm::make_unique<UnitDataConsumerActionImpl>(
+      std::move(IndexOpts), ConsumerFactory);
+  if (WrappedAction)
+    return llvm::make_unique<WrappingIndexAction>(std::move(WrappedAction),
+                                                  std::move(ActionImpl));
+  return llvm::make_unique<IndexAction>(std::move(ActionImpl));
+};
+
+std::unique_ptr<FrontendAction> index::createIndexDataRecordingAction(
+    RecordingOptions RecordOpts,
+    std::unique_ptr<FrontendAction> WrappedAction) {
+
+  auto ConsumerFactory =
+      [RecordOpts](
+          UnitDetails UnitInfo) -> std::unique_ptr<UnitIndexDataConsumer> {
+    return llvm::make_unique<UnitIndexDataRecorder>(std::move(UnitInfo),
+                                                    RecordOpts);
+  };
+  return createUnitIndexingAction(ConsumerFactory, std::move(RecordOpts),
+                                  std::move(WrappedAction));
+};
+
+RecordingOptions
+index::getRecordingOptionsFromFrontendOptions(const FrontendOptions &FEOpts) {
+  RecordingOptions RecordOpts;
+  RecordOpts.DataDirPath = FEOpts.IndexStorePath;
+  if (FEOpts.IndexIgnoreSystemSymbols) {
+    RecordOpts.SystemSymbolFilter =
+        index::IndexingOptions::SystemSymbolFilterKind::None;
+  }
+  RecordOpts.RecordSymbolCodeGenName = FEOpts.IndexRecordCodegenName;
+  return RecordOpts;
+}
+
+void index::recordIndexDataForModuleFile(serialization::ModuleFile *ModFile,
+                                         RecordingOptions RecordOpts,
+                                         const CompilerInstance &CI) {
+  auto UnitConsumerFactory = [RecordOpts](UnitDetails UnitInfo) {
+    return llvm::make_unique<UnitIndexDataRecorder>(std::move(UnitInfo),
+                                                    RecordOpts);
+  };
+  return indexModuleFile(*ModFile, CI, UnitConsumerFactory,
+                         std::move(RecordOpts));
+}
diff --git a/lib/Index/IndexingContext.cpp b/lib/Index/IndexingContext.cpp
index de9fe39df0..17690487a1 100644
--- a/lib/Index/IndexingContext.cpp
+++ b/lib/Index/IndexingContext.cpp
@@ -8,14 +8,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "IndexingContext.h"
-#include "clang/Index/IndexDataConsumer.h"
 #include "clang/AST/ASTContext.h"
-#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Index/IndexDataConsumer.h"
+#include "llvm/Support/Path.h"
 
 using namespace clang;
 using namespace index;
+using namespace llvm;
 
 static bool isGeneratedDecl(const Decl *D) {
   if (auto *attr = D->getAttr<ExternalSourceSymbolAttr>()) {
@@ -24,6 +26,59 @@ static bool isGeneratedDecl(const Decl *D) {
   return false;
 }
 
+void SystemFileCache::setSysrootPath(llvm::StringRef Path) {
+  // Ignore sysroot path if it points to root, otherwise every header will be
+  // treated as system one.
+  SysrootPath = sys::path::root_path(Path) == Path ? StringRef() : Path;
+
+  // Invalidate existing results
+  LastFileCheck = {FileID(), false};
+  DirEntries.clear();
+}
+
+SystemFileCache::SystemFileCache(std::string Path) { setSysrootPath(Path); }
+
+bool SystemFileCache::isSystem(clang::FileID FID, clang::SourceManager &SM) {
+  if (LastFileCheck.first == FID)
+    return LastFileCheck.second;
+
+  auto Result = [&](bool Res) -> bool {
+    LastFileCheck = {FID, Res};
+    return Res;
+  };
+
+  bool Invalid = false;
+  const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid);
+  if (Invalid || !SEntry.isFile())
+    return Result(false);
+
+  const SrcMgr::FileInfo &FI = SEntry.getFile();
+  if (FI.getFileCharacteristic() != SrcMgr::C_User)
+    return Result(true);
+
+  auto *CC = FI.getContentCache();
+  if (!CC)
+    return Result(false);
+  auto *FE = CC->OrigEntry;
+  if (!FE)
+    return Result(false);
+
+  if (SysrootPath.empty())
+    return Result(false);
+
+  // Check if directory is in sysroot so that we can consider system headers
+  // even the headers found via a user framework search path, pointing inside
+  // sysroot.
+  auto DirEntry = FE->getDir();
+  auto Pair = DirEntries.insert(std::make_pair(DirEntry, false));
+  bool &IsSystemDir = Pair.first->second;
+  bool WasInserted = Pair.second;
+  if (WasInserted) {
+    IsSystemDir = StringRef(DirEntry->getName()).startswith(SysrootPath);
+  }
+  return Result(IsSystemDir);
+}
+
 bool IndexingContext::shouldIndex(const Decl *D) {
   return !isGeneratedDecl(D);
 }
@@ -93,12 +148,8 @@ bool IndexingContext::importedModule(const ImportDecl *ImportD) {
   if (FID.isInvalid())
     return true;
 
-  bool Invalid = false;
-  const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid);
-  if (Invalid || !SEntry.isFile())
-    return true;
-
-  if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) {
+  bool IsInSystemFile = SystemCache.isSystem(FID, SM);
+  if (IsInSystemFile) {
     switch (IndexOpts.SystemSymbolFilter) {
     case IndexingOptions::SystemSymbolFilterKind::None:
       return true;
@@ -112,7 +163,8 @@ bool IndexingContext::importedModule(const ImportDecl *ImportD) {
   if (ImportD->isImplicit())
     Roles |= (unsigned)SymbolRole::Implicit;
 
-  return DataConsumer.handleModuleOccurence(ImportD, Roles, FID, Offset);
+  return DataConsumer.handleModuleOccurence(ImportD, Roles, FID, Offset,
+                                            IsInSystemFile);
 }
 
 bool IndexingContext::isTemplateImplicitInstantiation(const Decl *D) {
@@ -205,7 +257,8 @@ static const Decl *adjustTemplateImplicitInstantiation(const Decl *D) {
   return nullptr;
 }
 
-static bool isDeclADefinition(const Decl *D, const DeclContext *ContainerDC, ASTContext &Ctx) {
+static bool isDeclADefinition(const Decl *D, const DeclContext *ContainerDC,
+                              ASTContext &Ctx) {
   if (auto VD = dyn_cast<VarDecl>(D))
     return VD->isThisDeclarationADefinition(Ctx);
 
@@ -321,7 +374,7 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc,
                                            const Expr *OrigE,
                                            const Decl *OrigD,
                                            const DeclContext *ContainerDC) {
-  if (D->isImplicit() && !isa<ObjCMethodDecl>(D))
+  if (D->isImplicit() && !(isa<ObjCMethodDecl>(D) || isa<ObjCIvarDecl>(D)))
     return true;
   if (!isa<NamedDecl>(D) || shouldSkipNamelessDecl(cast<NamedDecl>(D)))
     return true;
@@ -337,12 +390,8 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc,
   if (FID.isInvalid())
     return true;
 
-  bool Invalid = false;
-  const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid);
-  if (Invalid || !SEntry.isFile())
-    return true;
-
-  if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) {
+  bool IsInSystemFile = SystemCache.isSystem(FID, SM);
+  if (IsInSystemFile) {
     switch (IndexOpts.SystemSymbolFilter) {
     case IndexingOptions::SystemSymbolFilterKind::None:
       return true;
@@ -416,5 +465,5 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc,
 
   IndexDataConsumer::ASTNodeInfo Node{ OrigE, OrigD, Parent, ContainerDC };
   return DataConsumer.handleDeclOccurence(D, Roles, FinalRelations, FID, Offset,
-                                          Node);
+                                          IsInSystemFile, Node);
 }
diff --git a/lib/Index/IndexingContext.h b/lib/Index/IndexingContext.h
index 566651c83a..33e90e85bd 100644
--- a/lib/Index/IndexingContext.h
+++ b/lib/Index/IndexingContext.h
@@ -11,9 +11,11 @@
 #define LLVM_CLANG_LIB_INDEX_INDEXINGCONTEXT_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Index/IndexSymbol.h"
 #include "clang/Index/IndexingAction.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
 
 namespace clang {
   class ASTContext;
@@ -29,25 +31,51 @@ namespace clang {
   class Stmt;
   class Expr;
   class TypeLoc;
-  class SourceLocation;
+  class DirectoryEntry;
 
 namespace index {
   class IndexDataConsumer;
 
+/// Tracks the current system root path and computes and caches whether a
+/// file is considered a system file or not
+class SystemFileCache {
+  std::string SysrootPath;
+  // Records whether a directory entry is system or not.
+  llvm::DenseMap<const DirectoryEntry *, bool> DirEntries;
+  // Keeps track of the last check for whether a FileID is system or
+  // not. This is used to speed up isSystemFile() call.
+  std::pair<FileID, bool> LastFileCheck;
+
+public:
+  SystemFileCache() = default;
+  SystemFileCache(std::string SysrootPath);
+
+  void setSysrootPath(StringRef path);
+  StringRef getSysrootPath() const { return SysrootPath; }
+  bool isSystem(FileID FID, SourceManager &SM);
+};
+
+/// Generates and reports indexing data to the provided \c IndexDataConsumer
+/// for any AST nodes passed to its various \c index* methods.
 class IndexingContext {
   IndexingOptions IndexOpts;
+  SystemFileCache SystemCache;
   IndexDataConsumer &DataConsumer;
   ASTContext *Ctx = nullptr;
 
 public:
   IndexingContext(IndexingOptions IndexOpts, IndexDataConsumer &DataConsumer)
-    : IndexOpts(IndexOpts), DataConsumer(DataConsumer) {}
+      : IndexOpts(IndexOpts), DataConsumer(DataConsumer) {}
 
   const IndexingOptions &getIndexOpts() const { return IndexOpts; }
+  SystemFileCache &getSystemCache() { return SystemCache; }
   IndexDataConsumer &getDataConsumer() { return DataConsumer; }
 
   void setASTContext(ASTContext &ctx) { Ctx = &ctx; }
 
+  void setSysrootPath(StringRef path) { SystemCache.setSysrootPath(path); }
+  StringRef getSysrootPath() const { return SystemCache.getSysrootPath(); }
+
   bool shouldIndex(const Decl *D);
 
   const LangOptions &getLangOpts() const;
diff --git a/lib/Index/UnitIndexDataRecorder.cpp b/lib/Index/UnitIndexDataRecorder.cpp
new file mode 100644
index 0000000000..f7f2afe843
--- /dev/null
+++ b/lib/Index/UnitIndexDataRecorder.cpp
@@ -0,0 +1,52 @@
+//===--- UnitIndexDataRecorder.cpp - Persist index data to the file system ===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "UnitIndexDataRecorder.h"
+#include "clang/Frontend/FrontendOptions.h"
+
+using namespace clang;
+using namespace clang::index;
+
+UnitIndexDataRecorder::UnitIndexDataRecorder(UnitDetails UnitDetails,
+                                             RecordingOptions RecordOpts)
+    : UnitInfo(UnitDetails) {
+  // TODO
+}
+
+void UnitIndexDataRecorder::handleFileDependency(const FileEntry *FE,
+                                                 bool IsSystem) {
+  // TODO
+}
+
+void UnitIndexDataRecorder::handleInclude(const FileEntry *Source,
+                                          unsigned int Line,
+                                          const FileEntry *Target) {
+  // TODO
+}
+
+void UnitIndexDataRecorder::handleModuleImport(
+    const serialization::ModuleFile &Mod, bool IsSystem) {
+  // TODO
+}
+
+bool UnitIndexDataRecorder::shouldIndexModuleDependency(
+    const serialization::ModuleFile &Mod) {
+  // TODO
+  return true;
+};
+
+bool UnitIndexDataRecorder::handleFileOccurrences(
+    FileID FID, ArrayRef<DeclOccurrence> Occurs, bool IsSystem) {
+  // TODO
+  return false;
+};
+
+void UnitIndexDataRecorder::finish(){
+    // TODO
+};
diff --git a/lib/Index/UnitIndexDataRecorder.h b/lib/Index/UnitIndexDataRecorder.h
new file mode 100644
index 0000000000..6f81d61270
--- /dev/null
+++ b/lib/Index/UnitIndexDataRecorder.h
@@ -0,0 +1,52 @@
+//===--- UnitIndexDataRecorder.h - Persist index data to the file system --===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_INDEX_UNITINDEXDATARECORDER_H
+#define LLVM_CLANG_LIB_INDEX_UNITINDEXDATARECORDER_H
+
+#include "clang/Index/RecordingAction.h"
+#include "clang/Index/UnitIndexDataConsumer.h"
+
+namespace clang {
+class DiagnosticsEngine;
+class FrontendOptions;
+
+namespace index {
+
+/// Persists the provided index data for a single translation unit out to the
+/// file system.
+class UnitIndexDataRecorder : public UnitIndexDataConsumer {
+protected:
+  UnitDetails UnitInfo;
+
+public:
+  UnitIndexDataRecorder(UnitDetails UnitInfo, RecordingOptions RecordOpts);
+
+  void handleFileDependency(const FileEntry *FE, bool IsSystem) override;
+
+  void handleInclude(const FileEntry *Source, unsigned Line,
+                     const FileEntry *Target) override;
+
+  void handleModuleImport(const serialization::ModuleFile &Mod,
+                          bool IsSystem) override;
+
+  bool
+  shouldIndexModuleDependency(const serialization::ModuleFile &Mod) override;
+
+  bool handleFileOccurrences(FileID FID,
+                             ArrayRef<DeclOccurrence> OccurrencesSortedByOffset,
+                             bool IsSystem) override;
+
+  void finish() override;
+};
+
+} // end namespace index
+} // end namespace clang
+
+#endif
diff --git a/test/Index/Core/Inputs/module/ModDep.h b/test/Index/Core/Inputs/module/ModDep.h
new file mode 100644
index 0000000000..e96ef5440f
--- /dev/null
+++ b/test/Index/Core/Inputs/module/ModDep.h
@@ -0,0 +1,3 @@
+#include "ModTop.h"
+
+void ModDep_func(ModTopStruct s);
diff --git a/test/Index/Core/Inputs/module/ModSystem.h b/test/Index/Core/Inputs/module/ModSystem.h
new file mode 100644
index 0000000000..0419f97804
--- /dev/null
+++ b/test/Index/Core/Inputs/module/ModSystem.h
@@ -0,0 +1,4 @@
+
+typedef struct {} ModSystemStruct;
+
+void ModSystem_func(void);
diff --git a/test/Index/Core/Inputs/module/ModTop.h b/test/Index/Core/Inputs/module/ModTop.h
new file mode 100644
index 0000000000..60c56868bb
--- /dev/null
+++ b/test/Index/Core/Inputs/module/ModTop.h
@@ -0,0 +1,4 @@
+
+typedef struct {} ModTopStruct;
+
+void ModTop_func(void);
diff --git a/test/Index/Core/Inputs/module/ModTopSub1.h b/test/Index/Core/Inputs/module/ModTopSub1.h
new file mode 100644
index 0000000000..e1e3cf3ec5
--- /dev/null
+++ b/test/Index/Core/Inputs/module/ModTopSub1.h
@@ -0,0 +1 @@
+void ModTopSub1_func(void);
diff --git a/test/Index/Core/Inputs/module/ModTopSub2.h b/test/Index/Core/Inputs/module/ModTopSub2.h
new file mode 100644
index 0000000000..39d37f12f0
--- /dev/null
+++ b/test/Index/Core/Inputs/module/ModTopSub2.h
@@ -0,0 +1 @@
+// This header has no symbols, intended to show up as file dependency.
diff --git a/test/Index/Core/Inputs/module/module.modulemap b/test/Index/Core/Inputs/module/module.modulemap
index a132562eaf..ee4bf12953 100644
--- a/test/Index/Core/Inputs/module/module.modulemap
+++ b/test/Index/Core/Inputs/module/module.modulemap
@@ -1 +1,13 @@
 module ModA { header "ModA.h" export * }
+module ModDep { header "ModDep.h" export * }
+module ModSystem [system] { header "ModSystem.h" export * }
+module ModTop {
+  header "ModTop.h"
+  export *
+  module Sub1 {
+    header "ModTopSub1.h"
+  }
+  module Sub2 {
+    header "ModTopSub2.h"
+  }
+}
diff --git a/test/Index/Core/Inputs/sys/system-head.h b/test/Index/Core/Inputs/sys/system-head.h
index df0e39ed86..b5d4291886 100644
--- a/test/Index/Core/Inputs/sys/system-head.h
+++ b/test/Index/Core/Inputs/sys/system-head.h
@@ -1,3 +1,20 @@
+// UNIT: index-system.mm.o
+// UNIT: is-system: 0
+// UNIT: is-module: 0
+// UNIT: has-main: 1
+// UNIT: main-path: {{.*}}index-system.mm
+// UNIT: out-file: {{.*}}index-system.mm.o
+// UNIT: is-debug: 1
+
+// UNIT: DEPEND START
+// UNIT: File | user | {{.*}}index-system.mm
+// UNIT: File | system | {{.*}}system-head.h
+// UNIT: DEPEND END (2)
+
+// UNIT: INCLUDE START
+// UNIT: {{.*}}index-system.mm:4 -> {{.*}}system-head.h
+// UNIT: INCLUDE END (1)
+
 // CHECK: [[@LINE+1]]:12 | class/ObjC | Base | [[Base_USR:.*]] | {{.*}} | Decl | rel: 0
 @interface Base
 @end
diff --git a/test/Index/Core/Inputs/transitive-include.h b/test/Index/Core/Inputs/transitive-include.h
new file mode 100644
index 0000000000..b6564d89bd
--- /dev/null
+++ b/test/Index/Core/Inputs/transitive-include.h
@@ -0,0 +1,6 @@
+#include "system-head.h"
+
+struct Point {
+  int x;
+  int y;
+};
diff --git a/test/Index/Core/external-source-symbol-attr.m b/test/Index/Core/external-source-symbol-attr.m
index cdc5296697..d5dc48dcc6 100644
--- a/test/Index/Core/external-source-symbol-attr.m
+++ b/test/Index/Core/external-source-symbol-attr.m
@@ -1,4 +1,5 @@
 // RUN: c-index-test core -print-source-symbols -- %s -target x86_64-apple-macosx10.7 | FileCheck %s
+// RUN: c-index-test core -print-source-unit -- %s -target x86_64-apple-macosx10.7 | FileCheck %s
 
 #define EXT_DECL(mod_name) __attribute__((external_source_symbol(language="Swift", defined_in=mod_name)))
 #define GEN_DECL(mod_name) __attribute__((external_source_symbol(language="Swift", defined_in=mod_name, generated_declaration)))
diff --git a/test/Index/Core/index-instantiated-source.cpp b/test/Index/Core/index-instantiated-source.cpp
index 7a810fbdf3..2002d781b4 100644
--- a/test/Index/Core/index-instantiated-source.cpp
+++ b/test/Index/Core/index-instantiated-source.cpp
@@ -1,4 +1,5 @@
 // RUN: c-index-test core -print-source-symbols -- %s -std=c++14 -target x86_64-apple-macosx10.7 | FileCheck %s
+// RUN: c-index-test core -print-source-unit -- %s -std=c++14 -target x86_64-apple-macosx10.7 | FileCheck %s
 // References to declarations in instantiations should be canonicalized:
 
 template<typename T>
diff --git a/test/Index/Core/index-source.mm b/test/Index/Core/index-source.mm
index 049a0bdaf6..9d969d8a3d 100644
--- a/test/Index/Core/index-source.mm
+++ b/test/Index/Core/index-source.mm
@@ -1,4 +1,5 @@
 // RUN: c-index-test core -print-source-symbols -- %s -target x86_64-apple-macosx10.7 | FileCheck %s
+// RUN: c-index-test core -print-source-unit -- %s -target x86_64-apple-macosx10.7 | FileCheck -check-prefixes=CHECK %s
 
 @interface MyCls
 @end
diff --git a/test/Index/Core/index-subkinds.m b/test/Index/Core/index-subkinds.m
index 5eea046721..4c391f0d7f 100644
--- a/test/Index/Core/index-subkinds.m
+++ b/test/Index/Core/index-subkinds.m
@@ -1,4 +1,5 @@
 // RUN: c-index-test core -print-source-symbols -- %s -target x86_64-apple-macosx10.7 | FileCheck %s
+// RUN: c-index-test core -print-source-unit -- %s -target x86_64-apple-macosx10.7 | FileCheck %s
 
 // CHECK: [[@LINE+1]]:12 | class/ObjC | XCTestCase | c:objc(cs)XCTestCase | _OBJC_CLASS_$_XCTestCase | Decl | rel: 0
 @interface XCTestCase
diff --git a/test/Index/Core/index-system.mm b/test/Index/Core/index-system.mm
index 2ad31fae71..5f64a29811 100644
--- a/test/Index/Core/index-system.mm
+++ b/test/Index/Core/index-system.mm
@@ -1,3 +1,4 @@
 // RUN: c-index-test core -print-source-symbols -- %s -isystem %S/Inputs/sys | FileCheck %S/Inputs/sys/system-head.h
+// RUN: c-index-test core -print-source-unit -- %s -isystem %S/Inputs/sys | FileCheck -check-prefixes=UNIT,CHECK %S/Inputs/sys/system-head.h
 
 #include "system-head.h"
diff --git a/test/Index/Core/index-unit.mm b/test/Index/Core/index-unit.mm
new file mode 100644
index 0000000000..3815f30ef7
--- /dev/null
+++ b/test/Index/Core/index-unit.mm
@@ -0,0 +1,135 @@
+// RUN: rm -rf %t.mcp
+// RUN: c-index-test core -print-source-unit -- -arch x86_64 -mmacosx-version-min=10.7 -c %s -o %t.o -isystem %S/Inputs/sys -fmodules -fmodules-cache-path=%t.mcp -Xclang -fdisable-module-hash -I %S/Inputs/module -I %S/Inputs | FileCheck %s
+
+@import ModDep;
+@import ModSystem;
+
+
+
+// CHECK: ModTop.pcm
+
+// CHECK: is-system: 0
+// CHECK: is-module: 1
+// CHECK: module-name: ModTop
+// CHECK: has-main: 0
+// CHECK: main-path: {{$}}
+// CHECK: out-file: {{.*}}/ModTop.pcm
+
+// CHECK: DEPEND START
+// CHECK: File | user | {{.*}}/Inputs/module/ModTopSub2.h
+// CHECK: File | user | {{.*}}/Inputs/module/ModTopSub1.h
+// CHECK: File | user | {{.*}}/Inputs/module/ModTop.h
+// CHECK: DEPEND END (3)
+
+// CHECK: INCLUDE START
+// CHECK: INCLUDE END (0)
+
+// CHECK: {{.*}}/Inputs/module/ModTop.h
+// CHECK: 2:9 | struct/C | <no-name> | c:{{.*}} | <no-cgname> | Def | rel: 0
+// CHECK: 2:19 | type-alias/C | ModTopStruct | [[ModTopStruct_USR:.*]] | <no-cgname> | Def | rel: 0
+// CHECK: 4:6 | function/C | ModTop_func | {{.*}} | __Z11ModTop_funcv | Decl | rel: 0
+
+// CHECK: {{.*}}/Inputs/module/ModTopSub1.h
+// CHECK: 1:6 | function/C | ModTopSub1_func | {{.*}} | __Z15ModTopSub1_funcv | Decl | rel: 0
+
+
+
+// CHECK: ModDep.pcm
+
+// CHECK: is-system: 0
+// CHECK: is-module: 1
+// CHECK: module-name: ModDep
+// CHECK: has-main: 0
+// CHECK: main-path: {{$}}
+// CHECK: out-file: {{.*}}/ModDep.pcm
+
+// CHECK: DEPEND START
+// CHECK: File | user | {{.*}}/Inputs/module/ModDep.h
+// CHECK: Module | user | {{.*}}/ModTop.pcm
+// CHECK: DEPEND END (2)
+
+// CHECK: INCLUDE START
+// CHECK: INCLUDE END (0)
+
+// CHECK: {{.*}}/Inputs/module/ModDep.h
+// CHECK: 3:6 | function/C | ModDep_func | [[ModDep_func_USR:.*]] | __Z11ModDep_func12ModTopStruct | Decl | rel: 0
+// CHECK: 3:18 | type-alias/C | ModTopStruct | [[ModTopStruct_USR]] | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK-NEXT: RelCont | ModDep_func | [[ModDep_func_USR]]
+
+
+
+// CHECK: ModSystem.pcm
+
+// CHECK: is-system: 1
+// CHECK: is-module: 1
+// CHECK: module-name: ModSystem
+// CHECK: has-main: 0
+// CHECK: main-path: {{$}}
+// CHECK: out-file: {{.*}}/ModSystem.pcm
+
+// CHECK: DEPEND START
+// CHECK: File | system | {{.*}}/Inputs/module/ModSystem.h
+// CHECK: DEPEND END (1)
+
+// CHECK: INCLUDE START
+// CHECK: INCLUDE END (0)
+
+// CHECK: {{.*}}/Inputs/module/ModSystem.h
+// CHECK: 2:9 | struct/C | <no-name> | {{.*}} | <no-cgname> | Def | rel: 0
+// CHECK: 2:19 | type-alias/C | ModSystemStruct | {{.*}} | <no-cgname> | Def | rel: 0
+// CHECK: 4:6 | function/C | ModSystem_func | {{.*}} | __Z14ModSystem_funcv | Decl | rel: 0
+
+
+
+// CHECK: index-unit.mm.o
+
+// CHECK: is-system: 0
+// CHECK: is-module: 0
+// CHECK: module-name: {{$}}
+// CHECK: has-main: 1
+// CHECK: main-path: {{.*}}/index-unit.mm
+// CHECK: out-file: {{.*}}/index-unit.mm.o
+
+// CHECK: DEPEND START
+// CHECK: File | user | {{.*}}/index-unit.mm
+// CHECK: File | user | {{.*}}/Inputs/module/module.modulemap
+// CHECK: File | user | {{.*}}/Inputs/transitive-include.h
+// CHECK: File | system | {{.*}}/Inputs/sys/system-head.h
+// CHECK: Module | user | {{.*}}/ModDep.pcm
+// CHECK: Module | system | {{.*}}/ModSystem.pcm
+// CHECK: DEPEND END (6)
+
+// CHECK: INCLUDE START
+// CHECK: {{.*}}index-unit.mm:[[@LINE+1]] -> {{.*}}/Inputs/transitive-include.h
+#include "transitive-include.h"
+// CHECK: {{.*}}/Inputs/transitive-include.h:1 -> {{.*}}/Inputs/sys/system-head.h
+// CHECK: INCLUDE END (2)
+
+// CHECK: {{.*}}/Inputs/transitive-include.h
+// CHECK: 3:8 | struct/C | Point | [[Point_USR:.*]] | <no-cgname> | Def | rel: 0
+// CHECK: 4:7 | field/C | x | {{.*}} | <no-cgname> | Def,RelChild | rel: 1
+// CHECK-NEXT: RelChild | Point | [[Point_USR]]
+// CHECK: 5:7 | field/C | y | {{.*}} | <no-cgname> | Def,RelChild | rel: 1
+// CHECK-NEXT: RelChild | Point | [[Point_USR]]
+
+// CHECK: {{.*}}/Inputs/sys/system-head.h
+// CHECK: 19:12 | class/ObjC | Base | [[Base_USR:.*]] | _OBJC_CLASS_$_Base | Decl | rel: 0
+// CHECK: 23:11 | protocol/ObjC | Prot1 | [[Prot1_USR:.*]] | <no-cgname> | Decl | rel: 0
+// CHECK: 29:11 | protocol/ObjC | Prot2 | [[Prot2_USR:.*]] | <no-cgname> | Decl | rel: 0
+// CHECK: 29:17 | protocol/ObjC | Prot1 | [[Prot1_USR]] | <no-cgname> | Ref,RelBase,RelCont | rel: 1
+// CHECK-NEXT: RelBase,RelCont | Prot2 | [[Prot2_USR]]
+// CHECK: 39:12 | class/ObjC | Sub | [[Sub_USR:.*]] | _OBJC_CLASS_$_Sub | Decl | rel: 0
+// CHECK: 39:18 | class/ObjC | Base | [[Base_USR]] | _OBJC_CLASS_$_Base | Ref,RelBase,RelCont | rel: 1
+// CHECK-NEXT: RelBase,RelCont | Sub | [[Sub_USR]]
+// CHECK: 39:23 | protocol/ObjC | Prot2 | [[Prot2_USR]] | <no-cgname> | Ref,RelBase,RelCont | rel: 1
+// CHECK-NEXT: RelBase,RelCont | Sub | [[Sub_USR]]
+// CHECK: 39:30 | protocol/ObjC | Prot1 | [[Prot1_USR]] | <no-cgname> | Ref,RelBase,RelCont | rel: 1
+// CHECK-NEXT: RelBase,RelCont | Sub | [[Sub_USR]]
+// CHECK: 41:8 | instance-method/ObjC | getit | {{.*}} | -[Sub getit] | Decl,Dyn,RelChild | rel: 1
+// CHECK-NEXT: RelChild | Sub | [[Sub_USR]]
+// CHECK: 45:7 | class/C++ | Cls | [[Cls_USR:.*]] | <no-cgname> | Def | rel: 0
+// CHECK: 50:7 | class/C++ | SubCls1 | [[SubCls1_USR:.*]] | <no-cgname> | Def | rel: 0
+// CHECK: 50:24 | class/C++ | Cls | [[Cls_USR]] | <no-cgname> | Ref,RelBase,RelCont | rel: 1
+// CHECK-NEXT: RelBase,RelCont | SubCls1 | [[SubCls1_USR]]
+// CHECK: 52:12 | field/C++ | f | {{.*}} | <no-cgname> | Def,RelChild | rel: 1
+// CHECK-NEXT: RelChild | SubCls1 | [[SubCls1_USR]]
diff --git a/test/Index/Store/assembly-invocation.c b/test/Index/Store/assembly-invocation.c
new file mode 100644
index 0000000000..1e619bb758
--- /dev/null
+++ b/test/Index/Store/assembly-invocation.c
@@ -0,0 +1,3 @@
+// Make sure it doesn't crash.
+// RUN: %clang -target x86_64-apple-macosx10.7 -S %s -o %t.s
+// RUN: %clang -target x86_64-apple-macosx10.7 -c %t.s -o %t.o -index-store-path %t.idx
diff --git a/tools/c-index-test/core_main.cpp b/tools/c-index-test/core_main.cpp
index c255f54ba6..805f21cc77 100644
--- a/tools/c-index-test/core_main.cpp
+++ b/tools/c-index-test/core_main.cpp
@@ -12,15 +12,17 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/FrontendAction.h"
-#include "clang/Index/IndexingAction.h"
+#include "clang/Index/CodegenNameGenerator.h"
 #include "clang/Index/IndexDataConsumer.h"
 #include "clang/Index/USRGeneration.h"
-#include "clang/Index/CodegenNameGenerator.h"
+#include "clang/Index/UnitIndexDataConsumer.h"
+#include "clang/Index/UnitIndexingAction.h"
 #include "clang/Serialization/ASTReader.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/PrettyStackTrace.h"
 
 using namespace clang;
 using namespace clang::index;
@@ -33,18 +35,20 @@ namespace {
 enum class ActionType {
   None,
   PrintSourceSymbols,
+  PrintSourceUnit,
 };
 
 namespace options {
 
 static cl::OptionCategory IndexTestCoreCategory("index-test-core options");
 
-static cl::opt<ActionType>
-Action(cl::desc("Action:"), cl::init(ActionType::None),
-       cl::values(
-          clEnumValN(ActionType::PrintSourceSymbols,
-                     "print-source-symbols", "Print symbols from source")),
-       cl::cat(IndexTestCoreCategory));
+static cl::opt<ActionType> Action(
+    cl::desc("Action:"), cl::init(ActionType::None),
+    cl::values(clEnumValN(ActionType::PrintSourceSymbols,
+                          "print-source-symbols", "Print symbols from source"),
+               clEnumValN(ActionType::PrintSourceUnit, "print-source-unit",
+                          "Print unit info from source")),
+    cl::cat(IndexTestCoreCategory));
 
 static cl::extrahelp MoreHelp(
   "\nAdd \"-- <compiler arguments>\" at the end to setup the compiler "
@@ -72,6 +76,42 @@ static void printSymbolInfo(SymbolInfo SymInfo, raw_ostream &OS);
 static void printSymbolNameAndUSR(const Decl *D, ASTContext &Ctx,
                                   raw_ostream &OS);
 
+static void printDeclOccurrence(const Decl *D, SymbolRoleSet Roles,
+                                ArrayRef<SymbolRelation> Relations, FileID FID,
+                                unsigned Offset, bool IsInSystemFile,
+                                CodegenNameGenerator &CGNameGen,
+                                raw_ostream &OS) {
+  ASTContext &Ctx = D->getASTContext();
+  SourceManager &SM = Ctx.getSourceManager();
+
+  unsigned Line = SM.getLineNumber(FID, Offset);
+  unsigned Col = SM.getColumnNumber(FID, Offset);
+  OS << Line << ':' << Col << " | ";
+
+  printSymbolInfo(getSymbolInfo(D), OS);
+  OS << " | ";
+
+  printSymbolNameAndUSR(D, Ctx, OS);
+  OS << " | ";
+
+  if (CGNameGen.writeName(D, OS))
+    OS << "<no-cgname>";
+  OS << " | ";
+
+  printSymbolRoles(Roles, OS);
+  OS << " | ";
+
+  OS << "rel: " << Relations.size() << '\n';
+
+  for (auto &SymRel : Relations) {
+    OS << '\t';
+    printSymbolRoles(SymRel.Roles, OS);
+    OS << " | ";
+    printSymbolNameAndUSR(SymRel.RelatedSymbol, Ctx, OS);
+    OS << '\n';
+  }
+}
+
 namespace {
 
 class PrintIndexDataConsumer : public IndexDataConsumer {
@@ -89,42 +129,17 @@ public:
   bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles,
                            ArrayRef<SymbolRelation> Relations,
                            FileID FID, unsigned Offset,
-                           ASTNodeInfo ASTNode) override {
+                           bool IsInSystemFile, ASTNodeInfo ASTNode) override {
     ASTContext &Ctx = D->getASTContext();
     SourceManager &SM = Ctx.getSourceManager();
 
-    unsigned Line = SM.getLineNumber(FID, Offset);
-    unsigned Col = SM.getColumnNumber(FID, Offset);
-    OS << Line << ':' << Col << " | ";
-
-    printSymbolInfo(getSymbolInfo(D), OS);
-    OS << " | ";
-
-    printSymbolNameAndUSR(D, Ctx, OS);
-    OS << " | ";
-
-    if (CGNameGen->writeName(D, OS))
-      OS << "<no-cgname>";
-    OS << " | ";
-
-    printSymbolRoles(Roles, OS);
-    OS << " | ";
-
-    OS << "rel: " << Relations.size() << '\n';
-
-    for (auto &SymRel : Relations) {
-      OS << '\t';
-      printSymbolRoles(SymRel.Roles, OS);
-      OS << " | ";
-      printSymbolNameAndUSR(SymRel.RelatedSymbol, Ctx, OS);
-      OS << '\n';
-    }
-
+    printDeclOccurrence(D, Roles, Relations, FID, Offset, IsInSystemFile,
+                        *CGNameGen, OS);
     return true;
   }
 
   bool handleModuleOccurence(const ImportDecl *ImportD, SymbolRoleSet Roles,
-                             FileID FID, unsigned Offset) override {
+                             FileID FID, unsigned Offset, bool IsInSystemFile) override {
     ASTContext &Ctx = ImportD->getASTContext();
     SourceManager &SM = Ctx.getSourceManager();
 
@@ -235,6 +250,156 @@ static bool printSourceSymbolsFromModule(StringRef modulePath,
   return false;
 }
 
+class PrintUnitDataConsumer : public UnitIndexDataConsumer {
+  struct Dependency {
+    enum Kind { File, Module };
+
+    Kind Kind;
+    std::string Name;
+    bool IsSystem;
+
+    void print(raw_ostream &OS) const {
+      switch (Kind) {
+      case File:
+        OS << "File";
+        break;
+      case Module:
+        OS << "Module";
+        break;
+      }
+      OS << " | " << (IsSystem ? "system" : "user") << " | " << Name << '\n';
+    }
+  };
+
+  struct Include {
+    std::string Source;
+    std::string Target;
+    unsigned Line;
+
+    void print(raw_ostream &OS) const {
+      OS << Source << ':' << Line << " -> " << Target << '\n';
+    }
+  };
+
+  struct FileOccurrences {
+    SourceManager &SM;
+    FileID FID;
+    bool IsSystem;
+    std::vector<DeclOccurrence> Occurrences;
+
+    void print(raw_ostream &OS, CodegenNameGenerator &CGNameGen) const {
+      const FileEntry *FE = SM.getFileEntryForID(FID);
+      OS << '\n' << FE->getName() << "\n----------\n";
+      for (const DeclOccurrence &Occur : Occurrences) {
+        printDeclOccurrence(Occur.Dcl, Occur.Roles, Occur.Relations, FID,
+                            Occur.Offset, IsSystem, CGNameGen, OS);
+      }
+    }
+  };
+
+  raw_ostream &OS;
+  UnitDetails UnitInfo;
+  const CompilerInstance &CI;
+  CodegenNameGenerator CGNameGen;
+  bool IndexModDependencies = true;
+  std::vector<Dependency> Dependencies;
+  std::vector<Include> Includes;
+  std::vector<FileOccurrences> FileOccurInfos;
+
+public:
+  PrintUnitDataConsumer(raw_ostream &OS, UnitDetails UnitInfo,
+                        bool IndexModDeps)
+      : OS(OS), UnitInfo(UnitInfo), CI(UnitInfo.CI),
+        CGNameGen(CI.getASTContext()), IndexModDependencies(IndexModDeps) {}
+
+  void handleFileDependency(const FileEntry *FE, bool IsSystem) override {
+    Dependencies.push_back({Dependency::File, FE->getName(), IsSystem});
+  }
+
+  void handleModuleImport(const serialization::ModuleFile &Mod,
+                          bool IsSystem) override {
+    Dependencies.push_back({Dependency::Module, Mod.FileName, IsSystem});
+  }
+
+  void handleInclude(const FileEntry *Source, unsigned Line,
+                     const FileEntry *Target) override {
+    Includes.push_back({Source->getName(), Target->getName(), Line});
+  }
+
+  bool
+  shouldIndexModuleDependency(const serialization::ModuleFile &Mod) override {
+    return IndexModDependencies;
+  }
+
+  bool handleFileOccurrences(FileID FID,
+                             ArrayRef<DeclOccurrence> OccurrencesSortedByOffset,
+                             bool IsSystem) override {
+    SourceManager &SM = CI.getASTContext().getSourceManager();
+    FileOccurInfos.push_back({SM, FID, IsSystem, OccurrencesSortedByOffset});
+    return false;
+  }
+
+  void finish() override {
+    OS << sys::path::filename(UnitInfo.OutputFile) << '\n'
+       << "----------\n"
+       << "is-system: " << UnitInfo.IsSystemUnit << '\n'
+       << "is-module: " << UnitInfo.IsModuleUnit << '\n'
+       << "module-name: " << UnitInfo.ModuleName << '\n'
+       << "has-main: " << !!UnitInfo.RootFile << '\n'
+       << "main-path: "
+       << (UnitInfo.RootFile ? UnitInfo.RootFile->getName() : "") << '\n'
+       << "out-file: " << UnitInfo.OutputFile << '\n'
+       << "target: " << UnitInfo.CI.getTargetOpts().Triple << '\n'
+       << "is-debug: " << UnitInfo.IsDebugCompilation << '\n';
+
+    OS << "\nDEPEND START\n";
+    for (const Dependency &Dep : Dependencies) {
+      Dep.print(OS);
+    }
+    OS << "DEPEND END (" << Dependencies.size() << ")\n";
+    OS << "\nINCLUDE START\n";
+    for (const Include &Inc : Includes) {
+      Inc.print(OS);
+    }
+    OS << "INCLUDE END (" << Includes.size() << ")\n";
+    for (const FileOccurrences &FileInfo : FileOccurInfos) {
+      FileInfo.print(OS, CGNameGen);
+    }
+    OS << '\n';
+  }
+};
+
+static bool printSourceUnit(ArrayRef<const char *> Args, bool IndexLocals,
+                            bool IndexModDeps) {
+  SmallVector<const char *, 4> ArgsWithProgName;
+  ArgsWithProgName.push_back("clang");
+  ArgsWithProgName.append(Args.begin(), Args.end());
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      CompilerInstance::createDiagnostics(new DiagnosticOptions));
+  auto CInvok = createInvocationFromCommandLine(ArgsWithProgName, Diags);
+  if (!CInvok)
+    return true;
+
+  raw_ostream &OS = outs();
+  UnitIndexingOptions IndexOpts;
+  IndexOpts.IndexFunctionLocals = IndexLocals;
+
+  auto ConsumerFactory = [&OS, IndexModDeps](UnitDetails UnitInfo) {
+    return llvm::make_unique<PrintUnitDataConsumer>(OS, std::move(UnitInfo),
+                                                    IndexModDeps);
+  };
+
+  std::unique_ptr<FrontendAction> IndexAction;
+  IndexAction = createUnitIndexingAction(ConsumerFactory, IndexOpts,
+                                         /*WrappedAction=*/nullptr);
+
+  auto PCHContainerOps = std::make_shared<PCHContainerOperations>();
+  std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCompilerInvocationAction(
+      std::move(CInvok), PCHContainerOps, Diags, IndexAction.get()));
+
+  return !Unit;
+}
+
 //===----------------------------------------------------------------------===//
 // Helper Utils
 //===----------------------------------------------------------------------===//
@@ -305,5 +470,15 @@ int indextest_core_main(int argc, const char **argv) {
     return printSourceSymbols(CompArgs, options::DumpModuleImports, options::IncludeLocals);
   }
 
+  if (options::Action == ActionType::PrintSourceUnit) {
+    if (CompArgs.empty()) {
+      errs()
+          << "error: missing compiler args; pass '-- <compiler arguments>'\n";
+      return 1;
+    }
+    return printSourceUnit(CompArgs, options::IncludeLocals,
+                           /*IndexModDepedencies=*/true);
+  }
+
   return 0;
 }
diff --git a/tools/diagtool/DiagnosticNames.cpp b/tools/diagtool/DiagnosticNames.cpp
index b0ca7f9806..0a3630d217 100644
--- a/tools/diagtool/DiagnosticNames.cpp
+++ b/tools/diagtool/DiagnosticNames.cpp
@@ -43,6 +43,7 @@ static const DiagnosticRecord BuiltinDiagnosticsByID[] = {
 #include "clang/Basic/DiagnosticSemaKinds.inc"
 #include "clang/Basic/DiagnosticAnalysisKinds.inc"
 #include "clang/Basic/DiagnosticRefactoringKinds.inc"
+#include "clang/Basic/DiagnosticIndexKinds.inc"
 #undef DIAG
 };
 
diff --git a/tools/libclang/CXIndexDataConsumer.cpp b/tools/libclang/CXIndexDataConsumer.cpp
index 89ac23be73..8df616573d 100644
--- a/tools/libclang/CXIndexDataConsumer.cpp
+++ b/tools/libclang/CXIndexDataConsumer.cpp
@@ -154,7 +154,7 @@ bool CXIndexDataConsumer::handleDeclOccurence(const Decl *D,
                                               SymbolRoleSet Roles,
                                              ArrayRef<SymbolRelation> Relations,
                                               FileID FID, unsigned Offset,
-                                              ASTNodeInfo ASTNode) {
+                                              bool IsInSystemFile, ASTNodeInfo ASTNode) {
   SourceLocation Loc = getASTContext().getSourceManager()
       .getLocForStartOfFile(FID).getLocWithOffset(Offset);
 
@@ -221,7 +221,7 @@ bool CXIndexDataConsumer::handleDeclOccurence(const Decl *D,
 bool CXIndexDataConsumer::handleModuleOccurence(const ImportDecl *ImportD,
                                                 SymbolRoleSet Roles,
                                                 FileID FID,
-                                                unsigned Offset) {
+                                                unsigned Offset, bool IsInSystemFile) {
   IndexingDeclVisitor(*this, SourceLocation(), nullptr).Visit(ImportD);
   return !shouldAbort();
 }
diff --git a/tools/libclang/CXIndexDataConsumer.h b/tools/libclang/CXIndexDataConsumer.h
index a54baadd07..b1cf0b1ea3 100644
--- a/tools/libclang/CXIndexDataConsumer.h
+++ b/tools/libclang/CXIndexDataConsumer.h
@@ -464,11 +464,11 @@ private:
   bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles,
                            ArrayRef<index::SymbolRelation> Relations,
                            FileID FID, unsigned Offset,
-                           ASTNodeInfo ASTNode) override;
+                           bool IsInSystemFile, ASTNodeInfo ASTNode) override;
 
   bool handleModuleOccurence(const ImportDecl *ImportD,
                              index::SymbolRoleSet Roles,
-                             FileID FID, unsigned Offset) override;
+                             FileID FID, unsigned Offset, bool IsInSystemFile) override;
 
   void finish() override;