another major rewrite

rank_hugo_themes.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # rank_hugo_themes.py
 
+from jinja2 import Environment, FileSystemLoader
 import re
 import toml
 from calendar import timegm
@@ -17,6 +18,9 @@ from sqlalchemy.orm import deferred, sessionmaker
 
 engine = create_engine('sqlite:///hugothemes.db', echo=False)
 Base = declarative_base()
+file_loader = FileSystemLoader('templates')
+env = Environment(loader=file_loader)
+template = env.get_template('base.html')
 
 
 class Tags(Base):
@@ -72,17 +76,19 @@ class Hugothemes(Base):
     tags_list = Column(TEXT)
     num_tags = Column(Integer)
     default_branch = Column(TEXT)
+    features_list = Column(TEXT)
+    num_features = Column(Integer)
 
     def __repr__(self):
         repr_string = "<(name = '%s', ETag = '%s', url = '%s', commit_sha = '%s', commit_date = '%s'"
         repr_string += ", commit_date_in_seconds = '%s', repo_ETag = '%s', stargazers_count = '%s', themes_toml_ETag = '%s'"
-        repr_string += ", themes_toml_content = '%s', tags_list = '%s', num_tags = '%s', default_branch = '%s')>"
+        repr_string += ", themes_toml_content = '%s', tags_list = '%s', num_tags = '%s', default_branch = '%s', features_list = '%s', num_features = '%s')>"
         repr_values = (
             self.name, self.ETag, self.url,
             self.commit_sha, self.commit_date, self.commit_date_in_seconds,
             self.repo_ETag, self.stargazers_count, self.themes_toml_ETag,
             self.themes_toml_content, self.tags_list,
-            self.num_tags, self.default_branch
+            self.num_tags, self.default_branch, self.features_list, self.num_features,
         )
         return repr_string % repr_values
 
@@ -384,9 +390,51 @@ def coalesce_themes():
         session.commit()
 
 
+def update_features_list_for_each_hugo_themes():
+    session = sessionmaker(bind=engine)()
+    themes = [theme[0] for theme in session.query(Hugothemes.name).all()]
+    match = re.compile(r'\s(\d+\.\d+\.\d+)\s')
+    for hugo_theme in themes:
+        theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
+        if theme.themes_toml_content is not None:
+            content = b64decode(theme.themes_toml_content).decode('utf-8')
+            theme_toml = toml.loads(match.sub(r'"\1"\n', content))
+            if 'features' in theme_toml:
+                if len(theme_toml['features']) > 0:
+                    theme_features = [feature.lower() for feature in theme_toml['features'] if len(feature) > 0]
+                    if theme.num_features != len(theme_features): theme.num_features = len(theme_features)
+                    if theme.num_features > 0:
+                        if theme.features_list != str(theme_features): theme.features_list = str(theme_features)
+                    else:
+                        if theme.features_list is not None: theme.features_list = None
+                else:
+                    if theme.features_list is not None: theme.features_list = None
+                    if theme.num_features != 0: theme.num_features = 0
+            else:
+                if theme.features_list is not None: theme.features_list = None
+                if theme.num_features != 0: theme.num_features = 0
+        else:
+            if theme.features_list is not None: theme.features_list = None
+            if theme.num_features != 0: theme.num_features = 0
+        session.commit()
+
+
+def get_corrected_tags(tags):
+    result = []
+    correct = True
+    for tag in tags:
+        if (len(tag) > 50): correct = False
+    if not correct:
+        for tag in tags:
+            result += [x.lstrip() for x in tag.split(',')]
+        return result
+    else:
+        return tags
+
+
 def update_tags_list_for_each_hugo_themes():
     session = sessionmaker(bind=engine)()
-    themes = [theme[0] for theme in session.query(Hugothemes.name).filter(Hugothemes.name != THEMESLISTREPO).all()]
+    themes = [theme[0] for theme in session.query(Hugothemes.name).all()]
     match = re.compile(r'\s(\d+\.\d+\.\d+)\s')
     for hugo_theme in themes:
         theme = session.query(Hugothemes).filter_by(name=hugo_theme).one()
@@ -398,7 +446,8 @@ def update_tags_list_for_each_hugo_themes():
             theme_toml = toml.loads(match.sub(r'"\1"\n', content))
             if 'tags' in theme_toml:
                 if len(theme_toml['tags']) > 0:
-                    theme_tags = [tag.lower() for tag in theme_toml['tags'] if len(tag) > 0]
+                    corrected_tags = get_corrected_tags(theme_toml['tags'])
+                    theme_tags = [tag.lower() for tag in corrected_tags if len(tag) > 0]
                     if theme.num_tags != len(theme_tags): theme.num_tags = len(theme_tags)
                     if theme.num_tags > 0:
                         if theme.tags_list != str(theme_tags): theme.tags_list = str(theme_tags)
@@ -519,7 +568,33 @@ def write_reports():
     by_date.close()
 
 
+def generate_report():
+    session = sessionmaker(bind=engine)()
+    hugo_themes = [
+        {
+            'name': theme.name,
+            'commit': theme.commit_sha[0:6],
+            'date': theme.commit_date[0:10],
+            'date_in_seconds': theme.commit_date_in_seconds,
+            'url': f'https://{theme.url}',
+            'short_name': theme.name.split('/')[1],
+            'num_stars': theme.stargazers_count,
+            'tags': literal_eval(theme.tags_list) if theme.tags_list is not None else [],
+            'features': literal_eval(theme.features_list) if theme.features_list is not None else [],
+        } for theme in session.query(Hugothemes).all()
+    ]
+    output = template.render(themes=hugo_themes)
+    index_page = open('hugo-themes-report/hugo-themes-report.html', 'w')
+    index_page.write(output)
+    index_page.close()
+
+
 if __name__ == "__main__":
+    '''
+    update_tags_list_for_each_hugo_themes()
+    update_features_list_for_each_hugo_themes()
+    generate_report()
+    '''
     get_hugo_themes_list()
     if len(THEMESLIST) > 300:
         clean_up()
@@ -534,5 +609,7 @@ if __name__ == "__main__":
         get_theme_dot_toml_for_each_hugo_themes_from_gitlab()
         coalesce_themes()
         update_tags_list_for_each_hugo_themes()
+        update_features_list_for_each_hugo_themes()
         update_tag_table()
-        write_reports()
+        # write_reports()
+        generate_report()

templates/base.html

@@ -0,0 +1,28 @@
+<!DOCTYPE html>
+<html lang='en'>
+  <head>
+    <title>>Hugo Themes Report</title>
+    <meta charset='utf-8'>
+    <meta name='viewport' content='width=device-width, initial-scale=1'>
+    <style>{% include 'css/main.css' %}</style>
+  </head>
+  <body>
+    <h1 id="title">
+      <a href="https://github.com/TrentSPalmer/hugo_themes_report" target="_blank">Hugo Themes Report</a>
+    </h1>
+    <button type="button" id="selection-button" class="collapsible">Select Tags/Features</button>
+    <div id="selection-menu">
+    </div>
+    <div id="results">
+    </div>
+  <script>
+    var themes = {{ themes }};
+    {% include 'js/selectionMenuCollapse.js' %}
+    {% include 'js/getAvailableTagsAndFeatures.js' %}
+    {% include 'js/buildSortByDiv.js' %}
+    {% include 'js/buildSelectionMenu.js' %}
+    {% include 'js/buildThemeTableRow.js' %}
+    {% include 'js/buildPage.js' %}
+  </script>
+  </body>
+</html>

templates/css/main.css

@@ -0,0 +1,78 @@
+body {
+  font-family: sans-serif;
+  width: 1200px;
+  max-width: 98%;
+  margin: 0 auto 0 auto;
+}
+
+#selection-menu {
+  display: none;
+}
+
+#title {
+  text-align: center;
+}
+
+.collapsible {
+  background-color: #eee;
+  color: #444;
+  cursor: pointer;
+  padding: 1.1rem;
+  width: 100%;
+  border: none;
+  text-align: left;
+  outline: none;
+  font-size: 1.1rem;
+}
+
+.active, .collapsible:hover {
+  background-color: #ccc;
+}
+
+.content {
+  padding: 0 1.1rem;
+  display: none;
+  overflow: hidden;
+  background-color: #f1f1f1;
+}
+
+.collapsible:after {
+  content: '\02795';
+  font-size: .8rem;
+  color: white;
+  float: right;
+  margin-left: .4rem;
+}
+
+.active:after {
+  content: '\2796';
+}
+
+#resultsTable {
+  width: 100%;
+  min-width: 500px;
+  border-spacing: 0;
+  padding: 2px;
+}
+
+#resultsTable tr{
+  height: 2rem;
+}
+
+#resultsTable tr:nth-child(even) {
+  background-color: #f2f2f2;
+}
+
+#themeTH {
+  text-align: left;
+}
+
+#results {
+  overflow-x: scroll;
+}
+
+td, th {
+  padding-left: 1rem;
+  padding-right: 1rem;
+  text-align: center;
+}

templates/js/buildPage.js

@@ -0,0 +1,84 @@
+function getSortBy() {
+  let sortByLastCommitInput = document.getElementById('sortByDate');
+  if (sortByLastCommitInput === null) {
+    return 'date';
+  } else {
+    return sortByLastCommitInput.checked ? 'date' : 'stars';
+  }
+}
+
+function getSortedThemes(themeList, sortedBy) {
+  if (sortedBy === 'date') {
+    return themeList.sort((a, b) => b.date_in_seconds - a.date_in_seconds);
+  } else {
+    return themeList.sort((a, b) => b.num_stars - a.num_stars);
+  }
+}
+
+function getSelectedTags() {
+  let tagSelectionInputs = document.getElementsByClassName('tagSelectionInput');
+  if (tagSelectionInputs.length > 0) {
+    return [...tagSelectionInputs].filter((x) => x.checked).map((y) => y.value);
+  } else {
+    return [];
+  }
+}
+
+function getSelectedFeatures() {
+  let featureSelectionInputs = document.getElementsByClassName('featureSelectionInput');
+  if (featureSelectionInputs.length > 0) {
+    return [...featureSelectionInputs].filter((x) => x.checked).map((y) => y.value);
+  } else {
+    return [];
+  }
+}
+
+function getFilteredThemes(selectedTags, selectedFeatures) {
+  if ((selectedTags.length === 0) && selectedFeatures.length === 0) {
+    return themes;
+  } else {
+    return themes
+      .filter((x) => selectedTags.every((y) => x.tags.includes(y)) )
+      .filter((z) => selectedFeatures.every((w) => z.features.includes(w)) );
+  }
+}
+
+function buildResults() {
+  let resultsDiv = document.getElementById('results');
+  resultsDiv.innerHTML = '';
+  let resultsTable = document.createElement("table");
+  resultsTable.id = 'resultsTable';
+  resultsTable.style.border = '1px solid black';
+  resultsTable.style.fontSize = '.9rem';
+
+  let resultsTableHeadRow = document.createElement("tr");
+  resultsDiv.appendChild(resultsTable);
+  resultsTable.appendChild(resultsTableHeadRow);
+
+  let themeTH = document.createElement("th");
+  themeTH.innerHTML = "theme";
+  resultsTableHeadRow.appendChild(themeTH);
+
+  let dateTH = document.createElement("th");
+  dateTH.innerHTML = "date";
+  resultsTableHeadRow.appendChild(dateTH);
+
+  let starsTH = document.createElement("th");
+  starsTH.innerHTML = "stars";
+  resultsTableHeadRow.appendChild(starsTH);
+
+  let commitTH = document.createElement("th");
+  commitTH.innerHTML = "commit";
+  resultsTableHeadRow.appendChild(commitTH);
+
+  let selectedTags = getSelectedTags();
+  let selectedFeatures = getSelectedFeatures();
+  let sortedBy = getSortBy();
+  let filtered_themes = getFilteredThemes(selectedTags, selectedFeatures);
+  let sorted_themes = getSortedThemes(filtered_themes, sortedBy);
+  sorted_themes.forEach(theme => addThemeTableRow(theme));
+
+  buildSelectionMenu(sorted_themes, sortedBy, selectedTags, selectedFeatures);
+};
+
+buildResults();

templates/js/buildSelectionMenu.js

@@ -0,0 +1,101 @@
+function buildTagSelectionInput(tag, selected, tagSelectionRow) {
+  let tagSelectionInputDiv = document.createElement('div');
+  tagSelectionInputDiv.style.width = '15rem';
+  tagSelectionInputDiv.style.maxWidth = '50%';
+  tagSelectionInputDiv.style.marginTop = '.5rem';
+  tagSelectionInputDiv.style.marginBottom = '.5rem';
+
+  let tagSelectionInput = document.createElement('input');
+  tagSelectionInput.type = "checkbox";
+  tagSelectionInput.id = tag.tag + "-selection-input";
+  tagSelectionInput.name = tag.tag + "-selection-input";
+  tagSelectionInput.value = tag.tag;
+  tagSelectionInput.checked = (selected) ? true : false;
+  tagSelectionInput.classList.add('tagSelectionInput');
+  tagSelectionInput.onclick = function() { buildResults(); };
+  tagSelectionInputDiv.appendChild(tagSelectionInput);
+
+  let tagSelectionInputLabel = document.createElement('label');
+  tagSelectionInputLabel.for = tag.tag + "-selection-input";
+  tagSelectionInputLabel.innerHTML = tag.tag + ' (' + tag.num_themes + ')';
+  tagSelectionInputDiv.appendChild(tagSelectionInputLabel);
+
+  tagSelectionRow.appendChild(tagSelectionInputDiv);
+}
+
+function buildTagSelectionDiv(selectedTags, availableTags) {
+  let selectionMenuDiv = document.getElementById('selection-menu');
+  let tagSelectionHeading = document.createElement('h2');
+  tagSelectionHeading.innerHTML = "Select Tags";
+  selectionMenuDiv.appendChild(tagSelectionHeading);
+
+  let tagSelectionRow = document.createElement('div');
+  tagSelectionRow.style.display = 'flex';
+  tagSelectionRow.style.flexWrap = 'wrap';
+  tagSelectionRow.style.justifyContent = 'space-around';
+
+  selectionMenuDiv.appendChild(tagSelectionRow);
+
+  availableTags
+    .filter((x) => selectedTags.includes(x.tag))
+    .forEach((y) => { buildTagSelectionInput(y, true, tagSelectionRow); });
+
+  availableTags
+    .filter((x) => !selectedTags.includes(x.tag))
+    .forEach((y) => { buildTagSelectionInput(y, false, tagSelectionRow); });
+}
+
+function buildFeatureSelectionInput(feature, selected, featureSelectionRow) {
+  let featureSelectionInputDiv = document.createElement('div');
+  featureSelectionInputDiv.style.width = '30rem';
+  featureSelectionInputDiv.style.maxWidth = '50%';
+  featureSelectionInputDiv.style.marginTop = '.5rem';
+  featureSelectionInputDiv.style.marginBottom = '.5rem';
+
+  let featureSelectionInput = document.createElement('input');
+  featureSelectionInput.type = "checkbox";
+  featureSelectionInput.id = feature.feature + "-selection-input";
+  featureSelectionInput.name = feature.feature + "-selection-input";
+  featureSelectionInput.value = feature.feature;
+  featureSelectionInput.checked = (selected) ? true : false;
+  featureSelectionInput.classList.add('featureSelectionInput');
+  featureSelectionInput.onclick = function() { buildResults(); };
+  featureSelectionInputDiv.appendChild(featureSelectionInput);
+
+  let featureSelectionInputLabel = document.createElement('label');
+  featureSelectionInputLabel.for =  feature.feature + "-selection-input";
+  featureSelectionInputLabel.innerHTML = feature.feature + ' (' + feature.num_themes + ')';
+  featureSelectionInputDiv.appendChild(featureSelectionInputLabel);
+
+  featureSelectionRow.appendChild(featureSelectionInputDiv);
+}
+
+function buildFeatureSelectionDiv(selectedFeatures, availableFeatures) {
+  let selectionMenuDiv = document.getElementById('selection-menu');
+  let featureSelectionHeading = document.createElement('h2');
+  featureSelectionHeading.innerHTML = "Select Features";
+  selectionMenuDiv.appendChild(featureSelectionHeading);
+
+  let featureSelectionRow = document.createElement('div');
+  featureSelectionRow.style.display = 'flex';
+  featureSelectionRow.style.flexWrap = 'wrap';
+  featureSelectionRow.style.justifyContent = 'space-around';
+
+  selectionMenuDiv.appendChild(featureSelectionRow);
+
+  availableFeatures
+    .filter((x) => selectedFeatures.includes(x.feature))
+    .forEach((y) => { buildFeatureSelectionInput(y, true, featureSelectionRow); });
+
+  availableFeatures
+    .filter((x) => !selectedFeatures.includes(x.feature))
+    .forEach((y) => { buildFeatureSelectionInput(y, false, featureSelectionRow); });
+}
+
+function buildSelectionMenu(sorted_themes, sortedBy, selectedTags, selectedFeatures) {
+  let availableTags = getAvailableTags(sorted_themes);
+  let availableFeatures = getAvailableFeatures(sorted_themes);
+  buildSortByDiv(sortedBy);
+  buildTagSelectionDiv(selectedTags, availableTags);
+  buildFeatureSelectionDiv(selectedFeatures, availableFeatures);
+}

templates/js/buildSortByDiv.js

@@ -0,0 +1,53 @@
+function buildSortByDiv(sortedBy) {
+  let menuDiv = document.getElementById('selection-menu');
+  menuDiv.innerHTML = '';
+  menuDiv.style.maxWidth = '100%';
+
+  let sortByRow = document.createElement('div');
+  sortByRow.id = 'sortByRow';
+  sortByRow.style.width = '500px';
+  sortByRow.style.maxWidth = '100%';
+  sortByRow.style.display = 'flex';
+  sortByRow.style.justifyContent = 'space-around';
+  sortByRow.style.margin = '1rem auto 1rem auto';
+
+  let sortByPrompt = document.createElement('div');
+  sortByPrompt.innerHTML = "Sort By:";
+  sortByRow.appendChild(sortByPrompt);
+
+  let sortByStarsDiv = document.createElement('div');
+  let sortByStarsInput = document.createElement('input');
+  sortByStarsInput.type = 'radio';
+  sortByStarsInput.id = 'sortByStars';
+  sortByStarsInput.name = 'sortBy';
+  sortByStarsInput.value = 'stars';
+  sortByStarsInput.checked = sortedBy === 'stars' ? true : false;
+  sortByStarsInput.onclick = function() { buildResults(); };
+  sortByStarsDiv.appendChild(sortByStarsInput);
+
+  let sortByStarsLabel = document.createElement('label');
+  sortByStarsLabel.for = 'stars';
+  sortByStarsLabel.innerHTML = 'Stars';
+  sortByStarsDiv.appendChild(sortByStarsLabel);
+
+  let sortByLastCommitDiv = document.createElement('div');
+  let sortByLastCommitInput = document.createElement('input');
+  sortByLastCommitInput.type = 'radio';
+  sortByLastCommitInput.id = 'sortByDate';
+  sortByLastCommitInput.name = 'sortBy';
+  sortByLastCommitInput.value = 'date';
+  sortByLastCommitInput.checked = sortedBy === 'date' ? true : false;
+  sortByLastCommitInput.onclick = function() { buildResults(); };
+  sortByLastCommitDiv.appendChild(sortByLastCommitInput);
+
+  let sortByLastCommitLabel = document.createElement('label');
+  sortByLastCommitLabel.for = 'date';
+  sortByLastCommitLabel.innerHTML = 'Latest Commit Date';
+  sortByLastCommitDiv.appendChild(sortByLastCommitLabel);
+
+
+  sortByRow.appendChild(sortByStarsDiv);
+  sortByRow.appendChild(sortByLastCommitDiv);
+
+  menuDiv.appendChild(sortByRow);
+}

templates/js/buildThemeTableRow.js

@@ -0,0 +1,28 @@
+function addThemeTableRow(theme) {
+  let resultsTable = document.getElementById('resultsTable');
+  let resultsTableRow = document.createElement("tr");
+
+  let themeTD = document.createElement("td");
+  themeTD.innerHTML = '<a href="' + theme.url + '" target="_blank">' + theme.short_name + '</a>';
+  themeTD.style.whiteSpace = 'nowrap';
+  themeTD.style.overFlow = 'hidden';
+  themeTD.style.width = '20%';
+  resultsTableRow.appendChild(themeTD);
+
+  let dateTD = document.createElement("td");
+  dateTD.innerHTML = theme.date;
+  dateTD.style.textAlign = 'center';
+  dateTD.style.minWidth = '8rem';
+  resultsTableRow.appendChild(dateTD);
+
+  let starsTD = document.createElement("td");
+  starsTD.innerHTML = theme.num_stars;
+  resultsTableRow.appendChild(starsTD);
+
+  let commitTD = document.createElement("td");
+  commitTD.innerHTML = theme.commit;
+  commitTD.style.minWidth = '7rem';
+  resultsTableRow.appendChild(commitTD);
+
+  resultsTable.appendChild(resultsTableRow);
+};

templates/js/getAvailableTagsAndFeatures.js

@@ -0,0 +1,48 @@
+function getAvailableFeatures(sorted_themes) {
+  let result = [];
+  sorted_themes.forEach(x => {
+    x.features.forEach(feature => {
+      if (result.length === 0) {
+        result.push({'feature': feature, 'num_themes': 1});
+      } else {
+        let features_in_result = result.map(y => y.feature);
+        if (features_in_result.includes(feature)) {
+          result.forEach(w => {
+            if (w.feature === feature) {
+              w.num_themes += 1;
+            }
+          });
+        } else {
+          result.push({'feature': feature, 'num_themes': 1});
+        }
+      }
+    });
+  });
+  // return result.sort((a, b) => a.feature.localeCompare(b.feature));
+  return result.sort((a, b) => b.num_themes - a.num_themes);
+}
+
+function getAvailableTags(sorted_themes) {
+  let result = [];
+  sorted_themes.forEach(x => {
+    x.tags.forEach(tag => {
+      if (result.length === 0) {
+        result.push({'tag': tag, 'num_themes': 1});
+      } else {
+        let tags_in_result = result.map(y => y.tag);
+        if (tags_in_result.includes(tag)) {
+          result.forEach(w => {
+            if (w.tag === tag) {
+              w.num_themes += 1;
+            }
+          });
+        } else {
+          result.push({'tag': tag, 'num_themes': 1});
+        }
+      }
+    });
+  });
+  return result.sort((a, b) => b.num_themes - a.num_themes);
+}
+
+

templates/js/selectionMenuCollapse.js

@@ -0,0 +1,11 @@
+var menuButton = document.getElementById("selection-button");
+
+menuButton.addEventListener("click", function() {
+  menuButton.classList.toggle("active");
+  var selectionMenu = document.getElementById("selection-menu");
+  if (selectionMenu.style.display === "block") {
+    selectionMenu.style.display = "none";
+  } else {
+    selectionMenu.style.display = "block";
+  }
+});