Refactored comment parsing to clang comments

src/common/visitor/comment/clang_visitor.cc

@@ -0,0 +1,221 @@
+/**
+ * src/common/visitor/comment/clang_visitor.cc
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "clang_visitor.h"
+
+namespace clanguml::common::visitor::comment {
+
+clang_visitor::clang_visitor(clang::SourceManager &source_manager)
+    : comment_visitor{source_manager}
+{
+}
+
+void clang_visitor::visit(
+    const clang::NamedDecl &decl, common::model::decorated_element &e)
+{
+    const auto *comment =
+        decl.getASTContext().getRawCommentForDeclNoCache(&decl);
+
+    if (comment == nullptr) {
+        return;
+    }
+
+    auto raw_comment = comment->getRawText(source_manager());
+
+    auto formatted_comment = comment->getFormattedText(
+        source_manager(), decl.getASTContext().getDiagnostics());
+
+    common::model::comment_t cmt = inja::json::object();
+    cmt["raw"] = raw_comment;
+    cmt["formatted"] = formatted_comment;
+
+    using clang::comments::BlockCommandComment;
+    using clang::comments::Comment;
+    using clang::comments::FullComment;
+    using clang::comments::ParagraphComment;
+    using clang::comments::ParamCommandComment;
+    using clang::comments::TextComment;
+    using clang::comments::TParamCommandComment;
+
+    FullComment *full_comment =
+        comment->parse(decl.getASTContext(), nullptr, &decl);
+
+    const auto &traits = decl.getASTContext().getCommentCommandTraits();
+
+    for (const auto *block : full_comment->getBlocks()) {
+        const auto block_kind = block->getCommentKind();
+        if (block_kind == Comment::ParagraphCommentKind) {
+            std::string paragraph_text;
+
+            visit_paragraph(clang::dyn_cast<ParagraphComment>(block), traits,
+                paragraph_text);
+            if (!cmt.contains("text"))
+                cmt["text"] = "";
+
+            cmt["text"] =
+                cmt["text"].get<std::string>() + "\n" + paragraph_text;
+        }
+        else if (block_kind == Comment::TextCommentKind) {
+            // TODO
+        }
+        else if (block_kind == Comment::ParamCommandCommentKind) {
+            visit_param_command(
+                clang::dyn_cast<ParamCommandComment>(block), traits, cmt);
+        }
+        else if (block_kind == Comment::TParamCommandCommentKind) {
+            visit_tparam_command(
+                clang::dyn_cast<TParamCommandComment>(block), traits, cmt);
+        }
+        else if (block_kind == Comment::BlockCommandCommentKind) {
+            auto *command = clang::dyn_cast<BlockCommandComment>(block);
+            auto command_info = traits.getCommandInfo(command->getCommandID());
+
+            if (command_info->IsBlockCommand && command_info->NumArgs == 0) {
+                // Visit block command with a single text argument, e.g.:
+                //    \brief text
+                //    \todo text
+                //    ...
+                visit_block_command(command, traits, cmt);
+            }
+            else if (command_info->IsParamCommand) {
+                // Visit function param block:
+                //   \param arg text
+                visit_param_command(
+                    clang::dyn_cast<ParamCommandComment>(command), traits, cmt);
+            }
+            else if (command_info->IsTParamCommand) {
+                // Visit template param block:
+                //   \tparam typename text
+                visit_tparam_command(
+                    clang::dyn_cast<TParamCommandComment>(command), traits,
+                    cmt);
+            }
+        }
+    }
+    e.set_comment(cmt);
+}
+
+void clang_visitor::visit_block_command(
+    const clang::comments::BlockCommandComment *command,
+    const clang::comments::CommandTraits &traits, common::model::comment_t &cmt)
+{
+    using clang::comments::Comment;
+    using clang::comments::ParagraphComment;
+    using clang::comments::TextComment;
+
+    std::string command_text;
+
+    for (auto paragraph_it = command->child_begin();
+         paragraph_it != command->child_end(); ++paragraph_it) {
+
+        if ((*paragraph_it)->getCommentKind() ==
+            Comment::ParagraphCommentKind) {
+            visit_paragraph(clang::dyn_cast<ParagraphComment>(*paragraph_it),
+                traits, command_text);
+        }
+    }
+
+    const auto command_name = command->getCommandName(traits).str();
+    if (!command_text.empty()) {
+        if (!cmt.contains(command_name))
+            cmt[command_name] = inja::json::array();
+
+        cmt[command_name].push_back(command_text);
+    }
+}
+
+void clang_visitor::visit_param_command(
+    const clang::comments::ParamCommandComment *command,
+    const clang::comments::CommandTraits &traits, common::model::comment_t &cmt)
+{
+    using clang::comments::Comment;
+    using clang::comments::ParagraphComment;
+    using clang::comments::TextComment;
+
+    std::string description;
+
+    const auto name = command->getParamNameAsWritten().str();
+
+    for (auto it = command->child_begin(); it != command->child_end(); ++it) {
+
+        if ((*it)->getCommentKind() == Comment::ParagraphCommentKind) {
+            visit_paragraph(
+                clang::dyn_cast<ParagraphComment>(*it), traits, description);
+        }
+    }
+
+    if (!name.empty()) {
+        if (!cmt.contains("param"))
+            cmt["param"] = inja::json::array();
+
+        inja::json param = inja::json::object();
+        param["name"] = name;
+        param["description"] = description;
+        cmt["param"].push_back(std::move(param));
+    }
+}
+
+void clang_visitor::visit_tparam_command(
+    const clang::comments::TParamCommandComment *command,
+    const clang::comments::CommandTraits &traits, common::model::comment_t &cmt)
+{
+    using clang::comments::Comment;
+    using clang::comments::ParagraphComment;
+    using clang::comments::TextComment;
+
+    std::string description;
+
+    const auto name = command->getParamNameAsWritten().str();
+
+    for (auto it = command->child_begin(); it != command->child_end(); ++it) {
+        if ((*it)->getCommentKind() == Comment::ParagraphCommentKind) {
+            visit_paragraph(
+                clang::dyn_cast<ParagraphComment>(*it), traits, description);
+        }
+    }
+
+    if (!name.empty()) {
+        if (!cmt.contains("tparam"))
+            cmt["tparam"] = inja::json::array();
+
+        inja::json param = inja::json::object();
+        param["name"] = name;
+        param["description"] = description;
+        cmt["tparam"].push_back(std::move(param));
+    }
+}
+
+void clang_visitor::visit_paragraph(
+    const clang::comments::ParagraphComment *paragraph,
+    const clang::comments::CommandTraits &traits, std::string &text)
+{
+    using clang::comments::Comment;
+    using clang::comments::TextComment;
+
+    for (auto text_it = paragraph->child_begin();
+         text_it != paragraph->child_end(); ++text_it) {
+
+        if ((*text_it)->getCommentKind() == Comment::TextCommentKind) {
+            // Merge paragraph lines into a single string
+            text += clang::dyn_cast<TextComment>((*text_it))->getText();
+            text += "\n";
+        }
+    }
+}
+
+}

src/common/visitor/comment/clang_visitor.h

@@ -0,0 +1,54 @@
+/**
+ * src/common/visitor/comment/clang_visitor.h
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Comment.h>
+#include <clang/Basic/SourceManager.h>
+
+#include "comment_visitor.h"
+
+namespace clanguml::common::visitor::comment {
+
+class clang_visitor : public comment_visitor {
+public:
+    clang_visitor(clang::SourceManager &source_manager);
+
+    void visit(const clang::NamedDecl &decl,
+        common::model::decorated_element &e) override;
+
+private:
+    void visit_block_command(
+        const clang::comments::BlockCommandComment *command,
+        const clang::comments::CommandTraits &traits,
+        common::model::comment_t &cmt);
+
+    void visit_param_command(
+        const clang::comments::ParamCommandComment *command,
+        const clang::comments::CommandTraits &traits,
+        common::model::comment_t &cmt);
+
+    void visit_tparam_command(
+        const clang::comments::TParamCommandComment *command,
+        const clang::comments::CommandTraits &traits,
+        common::model::comment_t &cmt);
+
+    void visit_paragraph(const clang::comments::ParagraphComment *paragraph,
+        const clang::comments::CommandTraits &traits, std::string &text);
+};
+}

src/common/visitor/comment/comment_visitor.cc

@@ -0,0 +1,33 @@
+/**
+ * src/common/visitor/comment/comment_visitor.cc
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "comment_visitor.h"
+
+namespace clanguml::common::visitor::comment {
+
+comment_visitor::comment_visitor(clang::SourceManager &source_manager)
+    : source_manager_{source_manager}
+{
+}
+
+clang::SourceManager &comment_visitor::source_manager()
+{
+    return source_manager_;
+}
+
+}

src/common/visitor/comment/comment_visitor.h

@@ -0,0 +1,41 @@
+/**
+ * src/common/visitor/comment/comment_visitor.h
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <clang/AST/Comment.h>
+#include <clang/Basic/SourceManager.h>
+
+#include "common/model/decorated_element.h"
+
+namespace clanguml::common::visitor::comment {
+
+class comment_visitor {
+public:
+    comment_visitor(clang::SourceManager &source_manager);
+
+    virtual ~comment_visitor() = default;
+
+    virtual void visit(
+        const clang::NamedDecl &decl, common::model::decorated_element &e) = 0;
+
+    clang::SourceManager &source_manager();
+
+private:
+    clang::SourceManager &source_manager_;
+};
+}

src/common/visitor/comment/plain_visitor.cc

@@ -0,0 +1,50 @@
+/**
+ * src/common/visitor/comment/plain_visitor.cc
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "plain_visitor.h"
+
+namespace clanguml::common::visitor::comment {
+
+plain_visitor::plain_visitor(clang::SourceManager &source_manager)
+    : comment_visitor{source_manager}
+{
+}
+
+void plain_visitor::visit(
+    const clang::NamedDecl &decl, common::model::decorated_element &e)
+{
+    const auto *comment =
+        decl.getASTContext().getRawCommentForDeclNoCache(&decl);
+
+    if (comment == nullptr) {
+        return;
+    }
+
+    auto raw_comment = comment->getRawText(source_manager());
+
+    auto formatted_comment = comment->getFormattedText(
+        source_manager(), decl.getASTContext().getDiagnostics());
+
+    common::model::comment_t cmt = inja::json::object();
+    cmt["raw"] = raw_comment;
+    cmt["formatted"] = formatted_comment;
+
+    e.set_comment(cmt);
+}
+
+}

src/common/visitor/comment/plain_visitor.h

@@ -0,0 +1,35 @@
+/**
+ * src/common/visitor/comment/plain_visitor.h
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <clang/AST/ASTContext.h>
+#include <clang/AST/Comment.h>
+#include <clang/Basic/SourceManager.h>
+
+#include "comment_visitor.h"
+
+namespace clanguml::common::visitor::comment {
+
+class plain_visitor : public comment_visitor {
+public:
+    plain_visitor(clang::SourceManager &source_manager);
+
+    void visit(const clang::NamedDecl &decl,
+        common::model::decorated_element &e) override;
+};
+}

src/common/visitor/translation_unit_visitor.cc

@@ -0,0 +1,76 @@
+/**
+ * src/common/visitor/translation_unit_visitor.cc
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "translation_unit_visitor.h"
+
+#include "comment/clang_visitor.h"
+#include "comment/plain_visitor.h"
+
+namespace clanguml::common::visitor {
+
+translation_unit_visitor::translation_unit_visitor(
+    clang::SourceManager &sm, const clanguml::config::diagram &config)
+    : source_manager_{sm}
+    , config_{config}
+{
+    if (config.comment_parser() == config::comment_parser_t::plain) {
+        comment_visitor_ =
+            std::make_unique<comment::plain_visitor>(source_manager_);
+    }
+    else if (config.comment_parser() == config::comment_parser_t::clang) {
+        comment_visitor_ =
+            std::make_unique<comment::clang_visitor>(source_manager_);
+    }
+}
+
+clang::SourceManager &translation_unit_visitor::source_manager() const
+{
+    return source_manager_;
+}
+
+void translation_unit_visitor::process_comment(
+    const clang::NamedDecl &decl, clanguml::common::model::decorated_element &e)
+{
+
+    assert(comment_visitor_.get() != nullptr);
+
+    comment_visitor_->visit(decl, e);
+
+    const auto *comment =
+        decl.getASTContext().getRawCommentForDeclNoCache(&decl);
+
+    if (comment != nullptr) {
+        // Process clang-uml decorators in the comments
+        // TODO: Refactor to use standard block comments processable by clang
+        //       comments
+        e.add_decorators(decorators::parse(comment->getFormattedText(
+            source_manager_, decl.getASTContext().getDiagnostics())));
+    }
+}
+
+void translation_unit_visitor::set_source_location(
+    const clang::Decl &decl, clanguml::common::model::source_location &element)
+{
+    if (decl.getLocation().isValid()) {
+        element.set_file(source_manager_.getFilename(decl.getLocation()).str());
+        element.set_line(
+            source_manager_.getSpellingLineNumber(decl.getLocation()));
+    }
+}
+
+}

src/common/visitor/translation_unit_visitor.h

@@ -0,0 +1,62 @@
+/**
+ * src/common/visitor/translation_unit_visitor.h
+ *
+ * Copyright (c) 2021-2022 Bartek Kryza <bkryza@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "comment/comment_visitor.h"
+#include "config/config.h"
+
+#include <clang/AST/Comment.h>
+#include <clang/Basic/SourceManager.h>
+
+#include <deque>
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+
+namespace clanguml::common::visitor {
+
+using found_relationships_t =
+    std::vector<std::pair<clanguml::common::model::diagram_element::id_t,
+        common::model::relationship_t>>;
+
+class translation_unit_visitor {
+public:
+    explicit translation_unit_visitor(
+        clang::SourceManager &sm, const clanguml::config::diagram &config);
+
+    clang::SourceManager &source_manager() const;
+
+    void finalize();
+
+protected:
+    void set_source_location(const clang::Decl &decl,
+        clanguml::common::model::source_location &element);
+
+    void process_comment(const clang::NamedDecl &decl,
+        clanguml::common::model::decorated_element &e);
+
+private:
+    clang::SourceManager &source_manager_;
+
+    // Reference to diagram config
+    const clanguml::config::diagram &config_;
+
+    std::unique_ptr<comment::comment_visitor> comment_visitor_;
+};
+}