diff --git a/include/components/builder.hpp b/include/components/builder.hpp
index 2f5a6384..b5f98e78 100644
--- a/include/components/builder.hpp
+++ b/include/components/builder.hpp
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <map>
+#include <unordered_set>
 
 #include "common.hpp"
 #include "components/types.hpp"
@@ -62,7 +63,7 @@ class builder {
   string m_output;
 
   map<tags::syntaxtag, int> m_tags{};
-  map<tags::attribute, bool> m_attrs{};
+  std::unordered_set<tags::attribute> m_attrs{};
 };
 
 POLYBAR_NS_END
diff --git a/src/components/builder.cpp b/src/components/builder.cpp
index ec153d2d..8f66b14b 100644
--- a/src/components/builder.cpp
+++ b/src/components/builder.cpp
@@ -32,10 +32,6 @@ void builder::reset() {
   m_tags[syntaxtag::P] = 0;
 
   m_attrs.clear();
-  m_attrs[attribute::NONE] = false;
-  m_attrs[attribute::UNDERLINE] = false;
-  m_attrs[attribute::OVERLINE] = false;
-
   m_output.clear();
 }
 
@@ -411,11 +407,12 @@ void builder::tag_open(syntaxtag tag, const string& value) {
  * Insert directive to use given attribute unless already set
  */
 void builder::tag_open(attribute attr) {
-  if (m_attrs[attr]) {
+  // Don't emit activation tag if the attribute is already activated
+  if (m_attrs.count(attr) != 0) {
     return;
   }
 
-  m_attrs[attr] = true;
+  m_attrs.insert(attr);
 
   switch (attr) {
     case attribute::UNDERLINE:
@@ -467,12 +464,11 @@ void builder::tag_close(syntaxtag tag) {
  * Insert directive to remove given attribute if set
  */
 void builder::tag_close(attribute attr) {
-  if (!m_attrs[attr]) {
+  // Don't close activation tag if it wasn't activated
+  if (m_attrs.erase(attr) == 0) {
     return;
   }
 
-  m_attrs[attr] = false;
-
   switch (attr) {
     case attribute::UNDERLINE:
       append("%{-u}");