Unified token breaking logic: support for line comments.

Summary:
Added BreakableLineComment, moved common code from
BreakableBlockComment to newly added BreakableComment. As a side-effect of the
rewrite, found another problem with escaped newlines and had to change
code which removes trailing whitespace from line comments not to break after
this patch.

Reviewers: klimek, djasper

Reviewed By: klimek

CC: cfe-commits

Differential Revision: http://llvm-reviews.chandlerc.com/D682

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@179693 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 81 insertions, 67 deletions

diff --git a/lib/Format/BreakableToken.h b/lib/Format/BreakableToken.h
index 0609104a6f..c1303183d3 100644
--- a/lib/Format/BreakableToken.h
+++ b/lib/Format/BreakableToken.h
@@ -26,54 +26,58 @@ namespace format {
 
 class BreakableToken {
 public:
+  BreakableToken(const SourceManager &SourceMgr, const FormatToken &Tok,
+                 unsigned StartColumn)
+      : Tok(Tok), StartColumn(StartColumn),
+        TokenText(SourceMgr.getCharacterData(Tok.getStartOfNonWhitespace()),
+                  Tok.TokenLength) {}
   virtual ~BreakableToken() {}
   virtual unsigned getLineCount() const = 0;
-  virtual unsigned getLineSize(unsigned Index) = 0;
+  virtual unsigned getLineSize(unsigned Index) const = 0;
   virtual unsigned getLineLengthAfterSplit(unsigned LineIndex,
-                                           unsigned TailOffset) = 0;
-  virtual unsigned getPrefixLength() = 0;
-  virtual unsigned getSuffixLength(unsigned LineIndex) = 0;
+                                           unsigned TailOffset) const = 0;
 
   // Contains starting character index and length of split.
   typedef std::pair<StringRef::size_type, unsigned> Split;
   virtual Split getSplit(unsigned LineIndex, unsigned TailOffset,
-                         unsigned ColumnLimit) = 0;
+                         unsigned ColumnLimit) const = 0;
   virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
                            bool InPPDirective,
                            WhitespaceManager &Whitespaces) = 0;
   virtual void trimLine(unsigned LineIndex, unsigned TailOffset,
                         unsigned InPPDirective,
-                        WhitespaceManager &Whitespaces) = 0;
+                        WhitespaceManager &Whitespaces) {}
+protected:
+  const FormatToken &Tok;
+  unsigned StartColumn;
+  StringRef TokenText;
 };
 
 class BreakableStringLiteral : public BreakableToken {
 public:
-  BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn)
-      : Tok(Tok), StartColumn(StartColumn) {}
+  BreakableStringLiteral(const SourceManager &SourceMgr, const FormatToken &Tok,
+                         unsigned StartColumn)
+      : BreakableToken(SourceMgr, Tok, StartColumn) {
+    assert(TokenText.startswith("\"") && TokenText.endswith("\""));
+  }
 
   virtual unsigned getLineCount() const { return 1; }
 
-  virtual unsigned getLineSize(unsigned Index) {
+  virtual unsigned getLineSize(unsigned Index) const {
     return Tok.TokenLength - 2; // Should be in sync with getLine
   }
 
   virtual unsigned getLineLengthAfterSplit(unsigned LineIndex,
-                                           unsigned TailOffset) {
-    return getPrefixLength() + getLine(LineIndex).size() - TailOffset +
-           getSuffixLength(LineIndex);
+                                           unsigned TailOffset) const {
+    return getDecorationLength() + getLine().size() - TailOffset;
   }
 
-  virtual unsigned getPrefixLength() { return StartColumn + 1; }
-
-  virtual unsigned getSuffixLength(unsigned LineIndex) { return 1; }
-
   virtual Split getSplit(unsigned LineIndex, unsigned TailOffset,
-                         unsigned ColumnLimit) {
-    StringRef Text = getLine(LineIndex).substr(TailOffset);
-    unsigned DecorationLength = getPrefixLength() + getSuffixLength(0);
-    if (ColumnLimit <= DecorationLength)
+                         unsigned ColumnLimit) const {
+    StringRef Text = getLine().substr(TailOffset);
+    if (ColumnLimit <= getDecorationLength())
       return Split(StringRef::npos, 0);
-    unsigned MaxSplit = ColumnLimit - DecorationLength;
+    unsigned MaxSplit = ColumnLimit - getDecorationLength();
     assert(MaxSplit < Text.size());
     StringRef::size_type SpaceOffset = Text.rfind(' ', MaxSplit);
     if (SpaceOffset != StringRef::npos && SpaceOffset != 0)
@@ -91,22 +95,20 @@ public:
   virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
                            bool InPPDirective, WhitespaceManager &Whitespaces) {
     unsigned WhitespaceStartColumn = StartColumn + Split.first + 2;
-    Whitespaces.breakToken(Tok, TailOffset + Split.first + 1, Split.second,
+    Whitespaces.breakToken(Tok, 1 + TailOffset + Split.first, Split.second,
                            "\"", "\"", InPPDirective, StartColumn,
                            WhitespaceStartColumn);
   }
 
-  virtual void trimLine(unsigned LineIndex, unsigned TailOffset,
-                        unsigned InPPDirective,
-                        WhitespaceManager &Whitespaces) {}
-
 private:
-  StringRef getLine(unsigned Index) {
+  StringRef getLine() const {
     // Get string without quotes.
     // FIXME: Handle string prefixes.
-    return StringRef(Tok.Tok.getLiteralData() + 1, Tok.TokenLength - 2);
+    return TokenText.substr(1, TokenText.size() - 2);
   }
 
+  unsigned getDecorationLength() const { return StartColumn + 2; }
+
   static StringRef::size_type getStartOfCharacter(StringRef Text,
                                                   StringRef::size_type Offset) {
     StringRef::size_type NextEscape = Text.find('\\');
@@ -157,69 +159,81 @@ private:
     return I;
   }
 
-  const FormatToken &Tok;
-  unsigned StartColumn;
 };
 
-class BreakableBlockComment : public BreakableToken {
+class BreakableComment : public BreakableToken {
 public:
-  BreakableBlockComment(const SourceManager &SourceMgr,
-                        const AnnotatedToken &Token, unsigned StartColumn);
-
-  void alignLines(WhitespaceManager &Whitespaces);
-
-  virtual unsigned getLineCount() const { return Lines.size(); }
-
-  virtual unsigned getLineSize(unsigned Index) {
+  virtual unsigned getLineSize(unsigned Index) const {
     return getLine(Index).size();
   }
 
-  virtual unsigned getLineLengthAfterSplit(unsigned LineIndex,
-                                           unsigned TailOffset) {
-    unsigned ContentStartColumn = getPrefixLength();
-    if (TailOffset == 0 && LineIndex == 0)
-      ContentStartColumn = StartColumn + 2;
-    return ContentStartColumn + getLine(LineIndex).size() - TailOffset +
-           getSuffixLength(LineIndex);
-  }
-
-  virtual unsigned getPrefixLength() {
-    return IndentAtLineBreak + (NeedsStar ? 2 : 0);
-  }
+  virtual unsigned getLineCount() const { return Lines.size(); }
 
-  virtual unsigned getSuffixLength(unsigned LineIndex) {
-    if (LineIndex + 1 < Lines.size())
-      return 0;
-    return 2;
+  virtual unsigned getLineLengthAfterSplit(unsigned LineIndex,
+                                           unsigned TailOffset) const {
+    return getContentStartColumn(LineIndex, TailOffset) +
+           getLine(LineIndex).size() - TailOffset;
   }
 
   virtual Split getSplit(unsigned LineIndex, unsigned TailOffset,
-                         unsigned ColumnLimit);
-
+                         unsigned ColumnLimit) const;
   virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
                            bool InPPDirective, WhitespaceManager &Whitespaces);
 
-  virtual void trimLine(unsigned LineIndex, unsigned TailOffset,
-                        unsigned InPPDirective, WhitespaceManager &Whitespaces);
+protected:
+  BreakableComment(const SourceManager &SourceMgr, const FormatToken &Tok,
+                   unsigned StartColumn)
+      : BreakableToken(SourceMgr, Tok, StartColumn) {}
 
-private:
   // Get comment lines without /* */, common prefix and trailing whitespace.
   // Last line is not trimmed, as it is terminated by */, so its trailing
   // whitespace is not really trailing.
-  StringRef getLine(unsigned Index) {
+  StringRef getLine(unsigned Index) const {
     return Index < Lines.size() - 1 ? Lines[Index].rtrim() : Lines[Index];
   }
 
-  const FormatToken &Tok;
-  const unsigned StartColumn;
-  StringRef TokenText;
-  unsigned OriginalStartColumn;
-  unsigned CommonPrefixLength;
+  unsigned getContentStartColumn(unsigned LineIndex,
+                                 unsigned TailOffset) const {
+    return (TailOffset == 0 && LineIndex == 0)
+               ? StartColumn
+               : IndentAtLineBreak + Decoration.size();
+  }
+
   unsigned IndentAtLineBreak;
-  bool NeedsStar;
+  StringRef Decoration;
   SmallVector<StringRef, 16> Lines;
 };
 
+class BreakableBlockComment : public BreakableComment {
+public:
+  BreakableBlockComment(const SourceManager &SourceMgr,
+                        const AnnotatedToken &Token, unsigned StartColumn);
+
+  void alignLines(WhitespaceManager &Whitespaces);
+
+  virtual unsigned getLineLengthAfterSplit(unsigned LineIndex,
+                                           unsigned TailOffset) const {
+    return BreakableComment::getLineLengthAfterSplit(LineIndex, TailOffset) +
+           (LineIndex + 1 < Lines.size() ? 0 : 2);
+  }
+
+  virtual void trimLine(unsigned LineIndex, unsigned TailOffset,
+                        unsigned InPPDirective, WhitespaceManager &Whitespaces);
+
+private:
+  unsigned OriginalStartColumn;
+  unsigned CommonPrefixLength;
+};
+
+class BreakableLineComment : public BreakableComment {
+public:
+  BreakableLineComment(const SourceManager &SourceMgr,
+                       const AnnotatedToken &Token, unsigned StartColumn);
+
+private:
+  static StringRef getLineCommentPrefix(StringRef Comment);
+};
+
 } // namespace format
 } // namespace clang